@specbook/core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/assert.ts +17 -6
- package/src/config.ts +17 -2
- package/src/controls.ts +34 -9
- package/src/diff.ts +131 -0
- package/src/environment.ts +175 -0
- package/src/index.ts +14 -2
- package/src/mock.ts +80 -70
- package/src/oauth.ts +164 -0
- package/src/reporters/ci.ts +53 -0
- package/src/reporters/color.ts +34 -0
- package/src/reporters/envelope.ts +63 -0
- package/src/reporters/index.ts +139 -25
- package/src/reporters/list.ts +254 -0
- package/src/reporters/redact.ts +113 -0
- package/src/reporters/trace.ts +70 -0
- package/src/request.ts +20 -2
- package/src/runner/index.ts +1 -0
- package/src/runner/run.ts +113 -25
- package/src/runner/suite.ts +122 -0
- package/src/runner/transport.ts +29 -4
- package/src/secrets.ts +31 -2
- package/src/sequence.ts +4 -4
- package/src/spec-config.ts +211 -14
- package/src/types.ts +49 -32
package/src/request.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { deepMerge } from './template'
|
|
|
2
2
|
import type {
|
|
3
3
|
AnyOp,
|
|
4
4
|
Init,
|
|
5
|
+
MockHook,
|
|
5
6
|
Operation,
|
|
6
7
|
OpMeta,
|
|
7
8
|
OpTypes,
|
|
@@ -15,6 +16,7 @@ function make<T extends OpTypes, I extends Init, C>(
|
|
|
15
16
|
input: RequestInput<T, I, C>,
|
|
16
17
|
dataPatches: readonly unknown[],
|
|
17
18
|
varOverrides: Readonly<Record<string, unknown>> = {},
|
|
19
|
+
mockHook?: MockHook<T, I>,
|
|
18
20
|
): RequestDef<T, I, C> {
|
|
19
21
|
return {
|
|
20
22
|
kind: 'request',
|
|
@@ -22,6 +24,7 @@ function make<T extends OpTypes, I extends Init, C>(
|
|
|
22
24
|
input,
|
|
23
25
|
dataPatches,
|
|
24
26
|
varOverrides,
|
|
27
|
+
mockHook,
|
|
25
28
|
with<C2>(o: RequestOverrides<T, I, C2>) {
|
|
26
29
|
const next: RequestInput<T, any, any> = { ...input }
|
|
27
30
|
if (o.path) next.path = deepMerge(input.path as any, o.path) as any
|
|
@@ -29,16 +32,31 @@ function make<T extends OpTypes, I extends Init, C>(
|
|
|
29
32
|
if (o.headers) next.headers = { ...(input.headers ?? {}), ...o.headers }
|
|
30
33
|
if (o.capture) next.capture = { ...(input.capture ?? {}), ...o.capture } as any
|
|
31
34
|
if (o.assert) next.assert = o.assert
|
|
32
|
-
if (o.mock) next.mock = { ...(input.mock ?? {}), ...o.mock } as any
|
|
33
35
|
if (o.auth !== undefined) next.auth = o.auth
|
|
34
36
|
if (o.assertMode) next.assertMode = o.assertMode
|
|
37
|
+
if (o.tags) next.tags = [...new Set([...(input.tags ?? []), ...o.tags])]
|
|
35
38
|
const patches = o.data !== undefined ? [...dataPatches, o.data] : dataPatches
|
|
36
39
|
const overrides = o.vars ? { ...varOverrides, ...o.vars } : varOverrides
|
|
37
|
-
return make(op, next, patches, overrides) as RequestDef<T, I, C & C2>
|
|
40
|
+
return make(op, next, patches, overrides, mockHook) as RequestDef<T, I, C & C2>
|
|
38
41
|
},
|
|
39
42
|
}
|
|
40
43
|
}
|
|
41
44
|
|
|
45
|
+
/**
|
|
46
|
+
* How the `mock` environment answers this request, when the generated example (filled from the request) is not enough.
|
|
47
|
+
* Rarely needed; `req` and `res` are typed from the operation. Returns the request, so it is the file's one export:
|
|
48
|
+
*
|
|
49
|
+
* export default withMock(GetOrder.request({ ... }), (req, res) => {
|
|
50
|
+
* if (res.status === 200) res.data.status = 'placed'
|
|
51
|
+
* })
|
|
52
|
+
*/
|
|
53
|
+
export function withMock<T extends OpTypes, I extends Init, C>(
|
|
54
|
+
def: RequestDef<T, I, C>,
|
|
55
|
+
hook: MockHook<T, I>,
|
|
56
|
+
): RequestDef<T, I, C> {
|
|
57
|
+
return make(def.op, def.input, def.dataPatches, def.varOverrides, hook)
|
|
58
|
+
}
|
|
59
|
+
|
|
42
60
|
/** Generated code calls this once per operation. */
|
|
43
61
|
export function defineOperation<T extends OpTypes = AnyOp>(meta: OpMeta): Operation<T> {
|
|
44
62
|
return {
|
package/src/runner/index.ts
CHANGED
package/src/runner/run.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createAsserter } from '../assert'
|
|
2
2
|
import { ControlError, type Controls, describeControls, resolveControls } from '../controls'
|
|
3
3
|
import type { Example, ExampleLoader, FileReader } from '../example'
|
|
4
|
+
import type { OAuthSettings } from '../oauth'
|
|
4
5
|
import { MissingSecretsError, maskDeep } from '../secrets'
|
|
5
6
|
import type { Sequence, StepCtx } from '../sequence'
|
|
6
7
|
import {
|
|
@@ -22,7 +23,12 @@ import type {
|
|
|
22
23
|
VarUse,
|
|
23
24
|
} from '../types'
|
|
24
25
|
import { describeFiles, isBlob } from './encode'
|
|
25
|
-
import
|
|
26
|
+
import {
|
|
27
|
+
DEFAULT_TIMEOUT_MS,
|
|
28
|
+
type PreparedRequest,
|
|
29
|
+
type RawResponse,
|
|
30
|
+
type Transport,
|
|
31
|
+
} from './transport'
|
|
26
32
|
|
|
27
33
|
// ---------------- public options ----------------
|
|
28
34
|
|
|
@@ -40,10 +46,22 @@ export interface AuthProfile {
|
|
|
40
46
|
token?: (res: ApiResponse<number, any>) => Session
|
|
41
47
|
apply: { headers?: Record<string, string>; query?: Record<string, string> }
|
|
42
48
|
retryOn401?: boolean
|
|
49
|
+
/**
|
|
50
|
+
* an interactive sign-in the UI can offer (authorization code with PKCE); the runner ignores it.
|
|
51
|
+
* `authorizationUrl`, `tokenUrl`, `openIdConnectUrl` and `clientId` may use environment variables
|
|
52
|
+
* (`'{{issuer}}/token'`).
|
|
53
|
+
*/
|
|
54
|
+
oauth?: OAuthSettings
|
|
43
55
|
}
|
|
44
56
|
|
|
45
57
|
export type RunEvent =
|
|
46
|
-
| {
|
|
58
|
+
| {
|
|
59
|
+
type: 'run:start'
|
|
60
|
+
id: string
|
|
61
|
+
title?: string
|
|
62
|
+
kind?: 'request' | 'sequence'
|
|
63
|
+
tags?: string[]
|
|
64
|
+
}
|
|
47
65
|
| { type: 'step:start'; name: string }
|
|
48
66
|
| { type: 'auth:login'; profile: string }
|
|
49
67
|
| { type: 'request'; name: string; request: RenderedRequest }
|
|
@@ -57,21 +75,31 @@ export type RunEvent =
|
|
|
57
75
|
|
|
58
76
|
export interface RunOptions {
|
|
59
77
|
env: string
|
|
60
|
-
/** config.vars
|
|
78
|
+
/** config.vars, then environments[env].vars, then the environment's file. */
|
|
61
79
|
vars?: Record<string, unknown>
|
|
62
80
|
/** Highest-precedence overrides (UI vars panel, CLI --var). */
|
|
63
81
|
overrides?: Record<string, unknown>
|
|
64
|
-
/**
|
|
65
|
-
|
|
82
|
+
/** Flow input values: the ones a person would set (UI controls, CLI --input). */
|
|
83
|
+
inputs?: Record<string, unknown>
|
|
66
84
|
transport: Transport
|
|
67
85
|
/** Resolved secret values, keyed by secret name. Not needed in the mock env. */
|
|
68
86
|
secrets?: Record<string, string>
|
|
87
|
+
/**
|
|
88
|
+
* Refuse to send a request that may change data (anything but GET, HEAD and OPTIONS, a GraphQL mutation, or a request
|
|
89
|
+
* not marked `safe`). Auth logins are exempt. For runs against real environments that must not write.
|
|
90
|
+
*/
|
|
91
|
+
readOnly?: boolean
|
|
92
|
+
/**
|
|
93
|
+
* A request with no `assert` of its own passes only on a 2xx response (recorded as the assertion `status is 2xx`);
|
|
94
|
+
* one with an `assert` is judged by that alone. Off, a request with no `assert` passes whatever the status.
|
|
95
|
+
*/
|
|
96
|
+
implicitStatusCheck?: boolean
|
|
69
97
|
/** Leave `{{secrets.*}}` placeholders in the wire request (a trusted server substitutes them). */
|
|
70
98
|
deferSecrets?: boolean
|
|
71
99
|
auth?: Record<string, AuthProfile>
|
|
72
100
|
/** service -> default auth profile name. */
|
|
73
101
|
serviceAuth?: Record<string, string>
|
|
74
|
-
/** Called at `input()` steps; omit for headless (
|
|
102
|
+
/** Called at `input()` steps; omit for headless (inputs given > default > first option). */
|
|
75
103
|
onInput?: (req: {
|
|
76
104
|
name: string
|
|
77
105
|
controls: ReturnType<typeof describeControls>
|
|
@@ -88,6 +116,13 @@ export interface RunOptions {
|
|
|
88
116
|
example?: ExampleLoader
|
|
89
117
|
/** Reads files named by upload fields (path relative to the example's `dir`). */
|
|
90
118
|
readFile?: FileReader
|
|
119
|
+
/**
|
|
120
|
+
* How long each request waits for an answer, in milliseconds, when the request does not say (`timeoutMs` on the request
|
|
121
|
+
* wins). 0 waits for as long as it takes. Default: 30 seconds.
|
|
122
|
+
*/
|
|
123
|
+
timeoutMs?: number
|
|
124
|
+
/** The `mock` environment answers with this status (the UI's "Respond with"). */
|
|
125
|
+
mockStatus?: number
|
|
91
126
|
}
|
|
92
127
|
|
|
93
128
|
// ---------------- internals ----------------
|
|
@@ -219,8 +254,22 @@ function substitute<T>(value: T, rt: Runtime, profile?: string): T {
|
|
|
219
254
|
return walk(value) as T
|
|
220
255
|
}
|
|
221
256
|
|
|
257
|
+
/** Events are masked as they are emitted: a listener (a stream, a log) must never see a secret value. */
|
|
222
258
|
function emit(rt: Runtime, e: RunEvent) {
|
|
223
|
-
rt.o.onEvent?.(e)
|
|
259
|
+
rt.o.onEvent?.(rt.mask.size ? maskDeep(e, rt.mask) : e)
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const READ_METHODS = new Set(['GET', 'HEAD', 'OPTIONS'])
|
|
263
|
+
|
|
264
|
+
/** True when the request cannot change data: a read method, a GraphQL query, or one marked `safe`. */
|
|
265
|
+
function isReadOnly(def: RequestDef<any, any, any>, prepared: PreparedRequest): boolean {
|
|
266
|
+
if (def.input.safe) return true
|
|
267
|
+
if (READ_METHODS.has(prepared.method)) return true
|
|
268
|
+
if (prepared.encoding === 'graphql') {
|
|
269
|
+
const body = prepared.body as { query?: unknown } | undefined
|
|
270
|
+
return typeof body?.query === 'string' && !/(^|[\s}])mutation\b/.test(body.query)
|
|
271
|
+
}
|
|
272
|
+
return false
|
|
224
273
|
}
|
|
225
274
|
|
|
226
275
|
function profileFor(rt: Runtime, def: RequestDef<any, any, any>): string | undefined {
|
|
@@ -286,13 +335,13 @@ async function ensureSession(
|
|
|
286
335
|
|
|
287
336
|
/** What earlier steps and the sequence hand to a request. */
|
|
288
337
|
interface Flow {
|
|
289
|
-
|
|
338
|
+
inputs: Record<string, unknown>
|
|
290
339
|
captured: Record<string, unknown>
|
|
291
340
|
/** capture name -> the step that produced it */
|
|
292
341
|
from: Record<string, string>
|
|
293
342
|
}
|
|
294
343
|
|
|
295
|
-
const NO_FLOW: Flow = {
|
|
344
|
+
const NO_FLOW: Flow = { inputs: {}, captured: {}, from: {} }
|
|
296
345
|
|
|
297
346
|
interface ExecOut {
|
|
298
347
|
result: StepResult
|
|
@@ -325,7 +374,7 @@ async function execute(
|
|
|
325
374
|
const layers: [VarUse['source'], Record<string, unknown>][] = [
|
|
326
375
|
['config', rt.o.vars ?? {}],
|
|
327
376
|
['init', evalInit(def.input.init)],
|
|
328
|
-
['
|
|
377
|
+
['input', flow.inputs],
|
|
329
378
|
['capture', flow.captured],
|
|
330
379
|
['override', evalInit(def.varOverrides)],
|
|
331
380
|
['override', rt.o.overrides ?? {}],
|
|
@@ -385,12 +434,12 @@ async function execute(
|
|
|
385
434
|
encoding,
|
|
386
435
|
mediaType: encoding === 'binary' ? def.op.mediaType : undefined,
|
|
387
436
|
formArrays: def.op.formArrays,
|
|
388
|
-
timeoutMs: def.input.timeoutMs,
|
|
389
|
-
mockStatus: rt.o.env === 'mock' ?
|
|
437
|
+
timeoutMs: def.input.timeoutMs ?? rt.o.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
438
|
+
mockStatus: rt.o.env === 'mock' ? rt.o.mockStatus : undefined,
|
|
390
439
|
mock:
|
|
391
440
|
rt.o.env === 'mock'
|
|
392
441
|
? {
|
|
393
|
-
|
|
442
|
+
hook: def.mockHook,
|
|
394
443
|
vars: scope,
|
|
395
444
|
pathParams,
|
|
396
445
|
rawQuery: query,
|
|
@@ -409,6 +458,10 @@ async function execute(
|
|
|
409
458
|
profile,
|
|
410
459
|
}
|
|
411
460
|
emit(rt, { type: 'request', name, request: result.request })
|
|
461
|
+
if (rt.o.readOnly && kind !== 'auth' && !isReadOnly(def, prepared))
|
|
462
|
+
throw new StepError(
|
|
463
|
+
`Refusing ${prepared.method} ${def.op.path} in "${rt.o.env}": this is a --safe run, which only reads.`,
|
|
464
|
+
)
|
|
412
465
|
|
|
413
466
|
const send = async () => {
|
|
414
467
|
if (profile) await ensureSession(rt, profile, authResults)
|
|
@@ -455,6 +508,17 @@ async function execute(
|
|
|
455
508
|
for (const r of a.results) {
|
|
456
509
|
emit(rt, { type: r.pass ? 'assert:pass' : 'assert:fail', name, assertion: r })
|
|
457
510
|
}
|
|
511
|
+
} else if (rt.o.implicitStatusCheck && kind === 'request') {
|
|
512
|
+
const pass = response.status >= 200 && response.status < 300
|
|
513
|
+
const implicit: AssertionResult = {
|
|
514
|
+
label: 'status is 2xx',
|
|
515
|
+
operator: 'status',
|
|
516
|
+
pass,
|
|
517
|
+
actual: response.status,
|
|
518
|
+
expected: '2xx',
|
|
519
|
+
}
|
|
520
|
+
result.assertions = [implicit]
|
|
521
|
+
emit(rt, { type: pass ? 'assert:pass' : 'assert:fail', name, assertion: implicit })
|
|
458
522
|
}
|
|
459
523
|
if (result.status === 'pass' && result.assertions.some((r) => !r.pass)) result.status = 'fail'
|
|
460
524
|
} catch (e: any) {
|
|
@@ -580,7 +644,13 @@ export async function collectSecretNames(
|
|
|
580
644
|
}
|
|
581
645
|
|
|
582
646
|
function newRuntime(o: RunOptions): Runtime {
|
|
583
|
-
|
|
647
|
+
// every resolved secret is masked from the start, not only once it has been substituted into a request
|
|
648
|
+
return {
|
|
649
|
+
o,
|
|
650
|
+
sessions: new Map(),
|
|
651
|
+
inflight: new Map(),
|
|
652
|
+
mask: new Set(Object.values(o.secrets ?? {})),
|
|
653
|
+
}
|
|
584
654
|
}
|
|
585
655
|
|
|
586
656
|
export async function runRequest(
|
|
@@ -589,10 +659,22 @@ export async function runRequest(
|
|
|
589
659
|
): Promise<RunResult> {
|
|
590
660
|
const rt = newRuntime(o)
|
|
591
661
|
const t0 = o.now?.() ?? now0()
|
|
592
|
-
emit(rt, {
|
|
662
|
+
emit(rt, {
|
|
663
|
+
type: 'run:start',
|
|
664
|
+
id: def.op.id,
|
|
665
|
+
title: def.input.name ?? def.op.id,
|
|
666
|
+
kind: 'request',
|
|
667
|
+
tags: def.input.tags,
|
|
668
|
+
})
|
|
593
669
|
const name = def.input.name ?? def.op.id
|
|
594
670
|
const { result, authResults } = await execute(rt, def, name, NO_FLOW)
|
|
595
|
-
return finish(
|
|
671
|
+
return finish(
|
|
672
|
+
rt,
|
|
673
|
+
'request',
|
|
674
|
+
{ id: def.op.id, title: name, tags: def.input.tags },
|
|
675
|
+
[...authResults, result],
|
|
676
|
+
t0,
|
|
677
|
+
)
|
|
596
678
|
}
|
|
597
679
|
|
|
598
680
|
export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promise<RunResult> {
|
|
@@ -600,7 +682,13 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
600
682
|
rt.seqAuth = seq.meta.auth
|
|
601
683
|
rt.seqAssertMode = seq.meta.assertMode
|
|
602
684
|
const t0 = o.now?.() ?? now0()
|
|
603
|
-
emit(rt, {
|
|
685
|
+
emit(rt, {
|
|
686
|
+
type: 'run:start',
|
|
687
|
+
id: seq.meta.id,
|
|
688
|
+
title: seq.meta.title,
|
|
689
|
+
kind: 'sequence',
|
|
690
|
+
tags: seq.meta.tags,
|
|
691
|
+
})
|
|
604
692
|
const steps: StepResult[] = []
|
|
605
693
|
const captured: Record<string, any> = {}
|
|
606
694
|
const from: Record<string, string> = {}
|
|
@@ -608,12 +696,12 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
608
696
|
Object.assign(captured, values)
|
|
609
697
|
for (const k of Object.keys(values)) from[k] = step
|
|
610
698
|
}
|
|
611
|
-
let
|
|
699
|
+
let inputs: Record<string, unknown>
|
|
612
700
|
try {
|
|
613
|
-
|
|
701
|
+
inputs = resolveControls(seq.meta.inputs ?? {}, o.inputs)
|
|
614
702
|
} catch (e: any) {
|
|
615
703
|
steps.push({
|
|
616
|
-
name: '
|
|
704
|
+
name: 'inputs',
|
|
617
705
|
kind: 'input',
|
|
618
706
|
status: 'error',
|
|
619
707
|
assertions: [],
|
|
@@ -625,7 +713,7 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
625
713
|
}
|
|
626
714
|
const vars = o.vars ?? {}
|
|
627
715
|
const ctx = (): StepCtx<any, any> => ({
|
|
628
|
-
|
|
716
|
+
inputs,
|
|
629
717
|
captured,
|
|
630
718
|
vars: { ...vars, ...(o.overrides ?? {}) },
|
|
631
719
|
env: o.env,
|
|
@@ -649,7 +737,7 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
649
737
|
if (node.kind === 'step') {
|
|
650
738
|
const def = typeof node.def === 'function' ? node.def(ctx()) : node.def
|
|
651
739
|
const { result, authResults } = await execute(rt, def, node.name, {
|
|
652
|
-
|
|
740
|
+
inputs,
|
|
653
741
|
captured,
|
|
654
742
|
from,
|
|
655
743
|
})
|
|
@@ -672,7 +760,7 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
672
760
|
for (let i = 0; i < items.length; i++) {
|
|
673
761
|
const def = node.def(items[i], ctx(), i)
|
|
674
762
|
const { result, authResults } = await execute(rt, def, `${node.name} #${i + 1}`, {
|
|
675
|
-
|
|
763
|
+
inputs,
|
|
676
764
|
captured,
|
|
677
765
|
from,
|
|
678
766
|
})
|
|
@@ -697,9 +785,9 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
697
785
|
emit(rt, { type: 'input:required', name: node.name, controls: described })
|
|
698
786
|
const given = o.onInput
|
|
699
787
|
? await o.onInput({ name: node.name, controls: described })
|
|
700
|
-
: (o.
|
|
788
|
+
: (o.inputs ?? {})
|
|
701
789
|
const values = resolveControls(controls, given)
|
|
702
|
-
|
|
790
|
+
inputs = { ...inputs, ...values }
|
|
703
791
|
steps.push({
|
|
704
792
|
name: node.name,
|
|
705
793
|
kind: 'input',
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// Running a list of requests and flows the way `specbook test` does: in order, retrying a failure, stopping at a number of
|
|
2
|
+
// failures or at a global deadline. The CLI and the docs site's terminal share it, so what the page shows is what a
|
|
3
|
+
// terminal does. Nothing here touches files or the process.
|
|
4
|
+
|
|
5
|
+
import type { Colors } from '../reporters/color'
|
|
6
|
+
import type { Sequence } from '../sequence'
|
|
7
|
+
import type { RequestDef, RunResult } from '../types'
|
|
8
|
+
import { type RunOptions, runRequest, runSequence } from './run'
|
|
9
|
+
|
|
10
|
+
export interface SuiteTarget {
|
|
11
|
+
title: string
|
|
12
|
+
/** project file, shown in reports */
|
|
13
|
+
file?: string
|
|
14
|
+
tags: string[]
|
|
15
|
+
run: RequestDef<any, any, any> | Sequence<any, any>
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface SuiteOptions {
|
|
19
|
+
/** what every run is given; `signal` is the suite's own */
|
|
20
|
+
run: Omit<RunOptions, 'signal'>
|
|
21
|
+
/** extra tries for a failing target (0 = one try) */
|
|
22
|
+
retries?: number
|
|
23
|
+
/** stop after this many failed targets */
|
|
24
|
+
bail?: number
|
|
25
|
+
/** milliseconds after which no further target starts, and a running one is aborted */
|
|
26
|
+
globalTimeoutMs?: number
|
|
27
|
+
/** called before each try, e.g. to seed faker again */
|
|
28
|
+
beforeAttempt?: () => void
|
|
29
|
+
/** called when a failing target is about to be tried again */
|
|
30
|
+
onRetry?: (target: SuiteTarget, attempt: number, allowed: number) => void
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface SuiteResult {
|
|
34
|
+
results: RunResult[]
|
|
35
|
+
failures: number
|
|
36
|
+
/** targets that did not start */
|
|
37
|
+
notRun: number
|
|
38
|
+
/** why the suite stopped early; empty when it did not */
|
|
39
|
+
stopped: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function runSuite(targets: SuiteTarget[], o: SuiteOptions): Promise<SuiteResult> {
|
|
43
|
+
const controller = new AbortController()
|
|
44
|
+
const deadline = o.globalTimeoutMs
|
|
45
|
+
? setTimeout(() => controller.abort(), o.globalTimeoutMs)
|
|
46
|
+
: undefined
|
|
47
|
+
const results: RunResult[] = []
|
|
48
|
+
const allowed = 1 + Math.max(0, o.retries ?? 0)
|
|
49
|
+
let failures = 0
|
|
50
|
+
let notRun = 0
|
|
51
|
+
let stopped = ''
|
|
52
|
+
for (const [i, t] of targets.entries()) {
|
|
53
|
+
if (controller.signal.aborted) {
|
|
54
|
+
notRun = targets.length - i
|
|
55
|
+
stopped = `stopped: the global timeout of ${o.globalTimeoutMs}ms passed`
|
|
56
|
+
break
|
|
57
|
+
}
|
|
58
|
+
let result: RunResult | undefined
|
|
59
|
+
let attempt = 0
|
|
60
|
+
while (attempt < allowed) {
|
|
61
|
+
attempt++
|
|
62
|
+
o.beforeAttempt?.()
|
|
63
|
+
const opts = { ...o.run, signal: controller.signal }
|
|
64
|
+
result =
|
|
65
|
+
t.run.kind === 'sequence' ? await runSequence(t.run, opts) : await runRequest(t.run, opts)
|
|
66
|
+
if (result.status === 'pass' || controller.signal.aborted) break
|
|
67
|
+
if (attempt < allowed) o.onRetry?.(t, attempt + 1, allowed)
|
|
68
|
+
}
|
|
69
|
+
const final = result as RunResult
|
|
70
|
+
results.push({
|
|
71
|
+
...final,
|
|
72
|
+
file: t.file,
|
|
73
|
+
attempts: attempt,
|
|
74
|
+
flaky: final.status === 'pass' && attempt > 1,
|
|
75
|
+
tags: final.tags.length ? final.tags : t.tags,
|
|
76
|
+
})
|
|
77
|
+
if (final.status !== 'pass') failures++
|
|
78
|
+
if (o.bail && failures >= o.bail && i < targets.length - 1) {
|
|
79
|
+
notRun = targets.length - i - 1
|
|
80
|
+
stopped = `stopped after ${failures} failed ${failures === 1 ? 'flow' : 'flows'} (--bail)`
|
|
81
|
+
break
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (deadline) clearTimeout(deadline)
|
|
85
|
+
return { results, failures, notRun, stopped }
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** The line a run opens with: the tool and version, then what is being run and where. */
|
|
89
|
+
export function headerLine(
|
|
90
|
+
h: {
|
|
91
|
+
version: string
|
|
92
|
+
env: string
|
|
93
|
+
picked: number
|
|
94
|
+
total: number
|
|
95
|
+
/** `main@abc1234`, `bundle <sha> built <date>` */
|
|
96
|
+
where?: string[]
|
|
97
|
+
},
|
|
98
|
+
colors: Colors,
|
|
99
|
+
): string {
|
|
100
|
+
return `${colors.bold(`specbook test ${h.version}`)} ${colors.dim(
|
|
101
|
+
[
|
|
102
|
+
`env ${h.env}`,
|
|
103
|
+
`${h.picked}${h.picked < h.total ? `/${h.total}` : ''} tests`,
|
|
104
|
+
...(h.where ?? []),
|
|
105
|
+
].join(' · '),
|
|
106
|
+
)}\n\n`
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** What a retry looks like in a terminal. */
|
|
110
|
+
export const retryLine = (colors: Colors, title: string, attempt: number, allowed: number) =>
|
|
111
|
+
` ${colors.yellow('↻')} retrying ${title} (attempt ${attempt} of ${allowed})\n`
|
|
112
|
+
|
|
113
|
+
/** A bundle's assertions point into the bundled file, which tells a reader nothing. */
|
|
114
|
+
export function withoutLocations(results: RunResult[]): RunResult[] {
|
|
115
|
+
const strip = (steps: RunResult['steps']): RunResult['steps'] =>
|
|
116
|
+
steps.map((s) => ({
|
|
117
|
+
...s,
|
|
118
|
+
assertions: s.assertions.map((a) => ({ ...a, location: undefined })),
|
|
119
|
+
...(s.children ? { children: strip(s.children) } : {}),
|
|
120
|
+
}))
|
|
121
|
+
return results.map((r) => ({ ...r, steps: strip(r.steps) }))
|
|
122
|
+
}
|
package/src/runner/transport.ts
CHANGED
|
@@ -16,11 +16,11 @@ export interface PreparedRequest {
|
|
|
16
16
|
/** Array style per form field, from the spec's `encoding` object. */
|
|
17
17
|
formArrays?: Record<string, FormArrays>
|
|
18
18
|
timeoutMs?: number
|
|
19
|
-
/** Mock status the caller wants served (mock env only). */
|
|
19
|
+
/** Mock status the caller wants served, e.g. the UI's "Respond with" (mock env only). */
|
|
20
20
|
mockStatus?: number
|
|
21
21
|
/** In-process hints for the mock transport; never serialized or substituted. */
|
|
22
22
|
mock?: {
|
|
23
|
-
|
|
23
|
+
hook?: import('../types').MockHook<any, any>
|
|
24
24
|
vars: Record<string, unknown>
|
|
25
25
|
pathParams: Record<string, unknown>
|
|
26
26
|
rawQuery: Record<string, unknown>
|
|
@@ -46,6 +46,18 @@ export function buildUrl(base: string, req: PreparedRequest): string {
|
|
|
46
46
|
return `${base.replace(/\/+$/, '')}${req.url}${q ? `?${q}` : ''}`
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
/** A config `baseUrl` may be relative (`/api/v1`): it is resolved against the page. */
|
|
50
|
+
export function absoluteBase(baseUrl: string, pageHref?: string): string {
|
|
51
|
+
try {
|
|
52
|
+
return new URL(baseUrl, pageHref).toString()
|
|
53
|
+
} catch {
|
|
54
|
+
return baseUrl
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** How long a request waits for an answer when nothing says otherwise. */
|
|
59
|
+
export const DEFAULT_TIMEOUT_MS = 30_000
|
|
60
|
+
|
|
49
61
|
export interface FetchTransportOptions {
|
|
50
62
|
baseUrl: (service: string) => string
|
|
51
63
|
fetch?: typeof fetch
|
|
@@ -53,6 +65,8 @@ export interface FetchTransportOptions {
|
|
|
53
65
|
|
|
54
66
|
export function createFetchTransport(opts: FetchTransportOptions): Transport {
|
|
55
67
|
return async (req, signal) => {
|
|
68
|
+
// a run stopped before this request left: nothing is sent, and no 'abort' will ever be heard from a signal that has fired
|
|
69
|
+
if (signal?.aborted) throw new Error('Stopped before the API answered.')
|
|
56
70
|
const f = opts.fetch ?? fetch
|
|
57
71
|
const headers: Record<string, string> = { ...req.headers }
|
|
58
72
|
let body: BodyInit | undefined
|
|
@@ -61,10 +75,15 @@ export function createFetchTransport(opts: FetchTransportOptions): Transport {
|
|
|
61
75
|
body = enc.body
|
|
62
76
|
if (enc.contentType) headers['content-type'] ??= enc.contentType
|
|
63
77
|
}
|
|
64
|
-
if (req.mockStatus !== undefined) headers['x-specbook-mock-status'] = String(req.mockStatus)
|
|
65
78
|
|
|
66
79
|
const ctl = new AbortController()
|
|
67
|
-
|
|
80
|
+
let timedOut = false
|
|
81
|
+
const timer = req.timeoutMs
|
|
82
|
+
? setTimeout(() => {
|
|
83
|
+
timedOut = true
|
|
84
|
+
ctl.abort()
|
|
85
|
+
}, req.timeoutMs)
|
|
86
|
+
: undefined
|
|
68
87
|
signal?.addEventListener('abort', () => ctl.abort())
|
|
69
88
|
try {
|
|
70
89
|
const res = await f(buildUrl(opts.baseUrl(req.service), req), {
|
|
@@ -88,6 +107,12 @@ export function createFetchTransport(opts: FetchTransportOptions): Transport {
|
|
|
88
107
|
}
|
|
89
108
|
}
|
|
90
109
|
return { status: res.status, headers: outHeaders, text, body: parsed }
|
|
110
|
+
} catch (e) {
|
|
111
|
+
// an abort that the request's own timeout caused says so, not "This operation was aborted"
|
|
112
|
+
if (timedOut) throw new Error(`Timed out after ${req.timeoutMs}ms: no answer from the API.`)
|
|
113
|
+
// a person pressing Stop, or the run's own deadline: say so, not "This operation was aborted"
|
|
114
|
+
if (signal?.aborted) throw new Error('Stopped before the API answered.')
|
|
115
|
+
throw e
|
|
91
116
|
} finally {
|
|
92
117
|
if (timer) clearTimeout(timer)
|
|
93
118
|
}
|
package/src/secrets.ts
CHANGED
|
@@ -67,9 +67,38 @@ export async function resolveSecrets(
|
|
|
67
67
|
return out
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
/**
|
|
70
|
+
/** Secrets shorter than this are not masked: they would blank out ordinary text. */
|
|
71
|
+
const MIN_MASKED = 3
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A secret as it can appear in a URL, a JSON string or a Basic-auth style base64 value, besides itself: what a server
|
|
75
|
+
* echoes back or an error message quotes is not always the raw value.
|
|
76
|
+
*/
|
|
77
|
+
export function maskVariants(secret: string): string[] {
|
|
78
|
+
const out = new Set<string>([secret])
|
|
79
|
+
const add = (f: () => string) => {
|
|
80
|
+
try {
|
|
81
|
+
out.add(f())
|
|
82
|
+
} catch {
|
|
83
|
+
/* not encodable: the raw value is still masked */
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
add(() => encodeURIComponent(secret))
|
|
87
|
+
add(() => encodeURI(secret))
|
|
88
|
+
add(() => JSON.stringify(secret).slice(1, -1))
|
|
89
|
+
add(() => btoa(String.fromCharCode(...new TextEncoder().encode(secret))))
|
|
90
|
+
return [...out]
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Masks with `***`, longest first so a secret containing another is hidden whole. Every secret is masked in its
|
|
95
|
+
* raw, URL-encoded, JSON-escaped and base64 forms; values under 3 characters are skipped to avoid masking
|
|
96
|
+
* ordinary text.
|
|
97
|
+
*/
|
|
71
98
|
export function maskDeep<T>(value: T, secrets: Iterable<string>): T {
|
|
72
|
-
const list = [
|
|
99
|
+
const list = [
|
|
100
|
+
...new Set([...secrets].filter((s) => s.length >= MIN_MASKED).flatMap(maskVariants)),
|
|
101
|
+
].sort((a, b) => b.length - a.length)
|
|
73
102
|
if (list.length === 0) return value
|
|
74
103
|
const maskStr = (s: string) => list.reduce((acc, v) => acc.split(v).join('***'), s)
|
|
75
104
|
const walk = (v: unknown): unknown => {
|
package/src/sequence.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { Controls, InferControls } from './controls'
|
|
|
2
2
|
import type { Assert, AssertMode, AuthProfileName, RequestDef } from './types'
|
|
3
3
|
|
|
4
4
|
export interface StepCtx<P, C> {
|
|
5
|
-
|
|
5
|
+
inputs: P
|
|
6
6
|
captured: C
|
|
7
7
|
vars: Record<string, unknown>
|
|
8
8
|
env: string
|
|
@@ -34,7 +34,7 @@ export interface SequenceMeta {
|
|
|
34
34
|
tags?: string[]
|
|
35
35
|
/** Heading the flow is listed under in the catalog; flows without one are under "Default". */
|
|
36
36
|
group?: string
|
|
37
|
-
|
|
37
|
+
inputs?: Controls
|
|
38
38
|
auth?: AuthProfileName | false
|
|
39
39
|
assertMode?: AssertMode
|
|
40
40
|
continueOnFailure?: boolean
|
|
@@ -55,7 +55,7 @@ export interface Sequence<P = {}, C = {}> {
|
|
|
55
55
|
name: string,
|
|
56
56
|
fn: (ctx: StepCtx<P, C>, h: { assert: Assert }) => void | Promise<void>,
|
|
57
57
|
): Sequence<P, C>
|
|
58
|
-
/** Pause for user input; the chosen values merge into ctx.
|
|
58
|
+
/** Pause for user input; the chosen values merge into ctx.inputs. */
|
|
59
59
|
input<R extends Controls>(
|
|
60
60
|
name: string,
|
|
61
61
|
controls: (ctx: StepCtx<P, C>) => R,
|
|
@@ -77,7 +77,7 @@ function build(meta: SequenceMeta, nodes: readonly StepNode[]): Sequence<any, an
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
export function sequence<P extends Controls = {}>(
|
|
80
|
-
meta: Omit<SequenceMeta, '
|
|
80
|
+
meta: Omit<SequenceMeta, 'inputs'> & { inputs?: P },
|
|
81
81
|
): Sequence<InferControls<P>, {}> {
|
|
82
82
|
return build(meta, [])
|
|
83
83
|
}
|