@nifrajs/testing 4.0.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -4
- package/dist/adversarial.d.ts +6 -38
- package/dist/adversarial.d.ts.map +1 -1
- package/dist/adversarial.js +304 -83
- package/dist/adversarial.js.map +1 -1
- package/dist/agent-eval.d.ts +25 -8
- package/dist/agent-eval.d.ts.map +1 -1
- package/dist/agent-eval.js +11 -6
- package/dist/agent-eval.js.map +1 -1
- package/dist/certification.d.ts +11 -1
- package/dist/certification.d.ts.map +1 -1
- package/dist/certification.js +136 -49
- package/dist/certification.js.map +1 -1
- package/dist/cookies.d.ts +1 -1
- package/dist/cookies.d.ts.map +1 -1
- package/dist/cookies.js +5 -3
- package/dist/cookies.js.map +1 -1
- package/dist/data-conformance.d.ts.map +1 -1
- package/dist/data-conformance.js +20 -8
- package/dist/data-conformance.js.map +1 -1
- package/dist/incident.d.ts +21 -6
- package/dist/incident.d.ts.map +1 -1
- package/dist/incident.js +99 -23
- package/dist/incident.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/trajectory.d.ts.map +1 -1
- package/dist/trajectory.js +31 -21
- package/dist/trajectory.js.map +1 -1
- package/dist/zod.d.ts +2 -2
- package/package.json +7 -7
- package/src/adversarial.ts +378 -147
- package/src/agent-eval.ts +30 -8
- package/src/certification.ts +137 -45
- package/src/cookies.ts +4 -2
- package/src/data-conformance.ts +23 -7
- package/src/incident.ts +121 -20
- package/src/index.ts +1 -0
- package/src/trajectory.ts +28 -28
- package/src/zod.ts +2 -2
package/src/adversarial.ts
CHANGED
|
@@ -7,6 +7,14 @@
|
|
|
7
7
|
* responses are validated off the hot path as part of the same laboratory run.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import type {
|
|
11
|
+
ContractAuthContext,
|
|
12
|
+
ContractCaseContext,
|
|
13
|
+
ContractCaseKind,
|
|
14
|
+
ContractExpectedResponse,
|
|
15
|
+
ContractTarget,
|
|
16
|
+
ContractWitness,
|
|
17
|
+
} from "@nifrajs/core/assurance"
|
|
10
18
|
import {
|
|
11
19
|
compileRoutePattern,
|
|
12
20
|
type ParamConstraint,
|
|
@@ -15,10 +23,15 @@ import {
|
|
|
15
23
|
import {
|
|
16
24
|
type JsonSchema,
|
|
17
25
|
type ReflectedRoute,
|
|
26
|
+
reflectMounts,
|
|
18
27
|
reflectRoutes,
|
|
19
28
|
type SchemaReflection,
|
|
20
29
|
} from "@nifrajs/core/reflection"
|
|
21
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
autoReflectJsonSchema,
|
|
32
|
+
explainAutoReflectJsonSchema,
|
|
33
|
+
generateMockValue,
|
|
34
|
+
} from "@nifrajs/mock"
|
|
22
35
|
|
|
23
36
|
const DEFAULT_ORIGIN = "http://nifra.contract"
|
|
24
37
|
const DEFAULT_SEED = 0x4e_49_46_52
|
|
@@ -26,12 +39,40 @@ const DEFAULT_MAX_MUTATIONS = 32
|
|
|
26
39
|
const DEFAULT_MAX_SHRINK_ATTEMPTS = 12
|
|
27
40
|
const DEFAULT_MAX_WITNESS_BYTES = 256 * 1024
|
|
28
41
|
const NO_BODY_METHODS: ReadonlySet<string> = new Set(["GET", "HEAD"])
|
|
29
|
-
type HeaderSource = ConstructorParameters<typeof Headers>[0]
|
|
30
42
|
|
|
31
|
-
type InputTarget = "body" | "query"
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
43
|
+
type InputTarget = "headers" | "params" | "body" | "query"
|
|
44
|
+
|
|
45
|
+
/** In the order a route validates them, so the first input that cannot resolve is the one named. */
|
|
46
|
+
const INPUT_TARGETS: readonly InputTarget[] = ["headers", "params", "body", "query"]
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Header names the harness never invents or mutates: a credential belongs to the witness or
|
|
50
|
+
* `prepareRequest`, and the rest are owned by the fetch runtime.
|
|
51
|
+
*/
|
|
52
|
+
const CREDENTIAL_HEADER =
|
|
53
|
+
/^(?:authorization|proxy-authorization|cookie|x-api-key)$|-key$|token|secret|signature/
|
|
54
|
+
const RUNTIME_HEADERS: ReadonlySet<string> = new Set([
|
|
55
|
+
"connection",
|
|
56
|
+
"content-length",
|
|
57
|
+
"expect",
|
|
58
|
+
"host",
|
|
59
|
+
"keep-alive",
|
|
60
|
+
"proxy-connection",
|
|
61
|
+
"te",
|
|
62
|
+
"trailer",
|
|
63
|
+
"transfer-encoding",
|
|
64
|
+
"upgrade",
|
|
65
|
+
])
|
|
66
|
+
const HEADER_NAME = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/
|
|
67
|
+
|
|
68
|
+
export type {
|
|
69
|
+
ContractAuthContext,
|
|
70
|
+
ContractCaseContext,
|
|
71
|
+
ContractCaseKind,
|
|
72
|
+
ContractExpectedResponse,
|
|
73
|
+
ContractTarget,
|
|
74
|
+
ContractWitness,
|
|
75
|
+
}
|
|
35
76
|
|
|
36
77
|
/** Anything that exposes reflected routes and a Web-standard in-process fetch handler. */
|
|
37
78
|
export interface ContractTestApp {
|
|
@@ -45,38 +86,6 @@ export interface ContractRuntime {
|
|
|
45
86
|
fetch(request: Request): Response | Promise<Response>
|
|
46
87
|
}
|
|
47
88
|
|
|
48
|
-
/** A known-good request. Missing body/query values are synthesized from inspectable JSON Schema. */
|
|
49
|
-
export interface ContractWitness {
|
|
50
|
-
readonly params?: Readonly<Record<string, string>>
|
|
51
|
-
readonly query?: unknown
|
|
52
|
-
readonly body?: unknown
|
|
53
|
-
readonly headers?: HeaderSource
|
|
54
|
-
/** Explicit auth state for the prepareRequest hook; the harness never invents credentials. */
|
|
55
|
-
readonly auth?: ContractAuthContext
|
|
56
|
-
/** Expected response for a guard/error witness. Omit for the normal success witness. */
|
|
57
|
-
readonly expected?: ContractExpectedResponse
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export interface ContractExpectedResponse {
|
|
61
|
-
readonly status: number
|
|
62
|
-
readonly contentType?: "json" | "text"
|
|
63
|
-
readonly headers?: Readonly<Record<string, string>>
|
|
64
|
-
/** When present, the decoded body is compared byte-structure-wise to this value. */
|
|
65
|
-
readonly body?: unknown
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/** Stable context passed to request/rejection hooks. It contains no request payloads or secrets. */
|
|
69
|
-
export interface ContractCaseContext {
|
|
70
|
-
readonly seed: number
|
|
71
|
-
readonly caseId: string
|
|
72
|
-
readonly route: ReflectedRoute
|
|
73
|
-
readonly runtime: string
|
|
74
|
-
readonly kind: ContractCaseKind
|
|
75
|
-
readonly target: ContractTarget
|
|
76
|
-
readonly auth?: ContractAuthContext
|
|
77
|
-
readonly mutation?: string
|
|
78
|
-
}
|
|
79
|
-
|
|
80
89
|
export interface AdversarialContractOptions {
|
|
81
90
|
/** Base origin for generated requests. Default `http://nifra.contract`. */
|
|
82
91
|
readonly origin?: string
|
|
@@ -124,10 +133,10 @@ export interface AdversarialContractOptions {
|
|
|
124
133
|
* generation for that route. Fail-safe: throwing or returning undefined leaves the schema opaque
|
|
125
134
|
* (NO_WITNESS, as today). Ready-made for zod: `import { zodJsonSchema } from "@nifrajs/testing/zod"`.
|
|
126
135
|
*
|
|
127
|
-
* Defaults to `@nifrajs/mock`'s `autoReflectJsonSchema`, which
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
* every opaque schema to stay opaque.
|
|
136
|
+
* Defaults to `@nifrajs/mock`'s `autoReflectJsonSchema`, which reads the schema's own Standard JSON
|
|
137
|
+
* Schema converter (`~standard.jsonSchema`, zod 4.2+ and other libraries) and, for an older zod,
|
|
138
|
+
* converts via `z.toJSONSchema` - so these routes get witnesses and constraint-driven mutations with
|
|
139
|
+
* no wiring. Pass a hook to override, or `() => undefined` to force every opaque schema to stay opaque.
|
|
131
140
|
*/
|
|
132
141
|
readonly reflectJsonSchema?: (schema: unknown) => JsonSchema | undefined
|
|
133
142
|
}
|
|
@@ -233,10 +242,10 @@ interface RouteLaboratory {
|
|
|
233
242
|
readonly route: ReflectedRoute
|
|
234
243
|
readonly routeKey: string
|
|
235
244
|
readonly witness: ContractWitness
|
|
236
|
-
readonly
|
|
237
|
-
readonly
|
|
238
|
-
|
|
239
|
-
readonly
|
|
245
|
+
readonly inputs: Partial<Record<InputTarget, ResolvedInput>>
|
|
246
|
+
readonly mutations: Readonly<Record<InputTarget, readonly Mutation[]>>
|
|
247
|
+
/** Every declared input, cookies included, has a value its validator accepts. */
|
|
248
|
+
readonly resolved: boolean
|
|
240
249
|
}
|
|
241
250
|
|
|
242
251
|
const recordOf = (value: unknown): Readonly<Record<string, unknown>> | undefined =>
|
|
@@ -482,19 +491,32 @@ async function rejectedByValidator(reflection: SchemaReflection, value: unknown)
|
|
|
482
491
|
return result.issues !== undefined
|
|
483
492
|
}
|
|
484
493
|
|
|
494
|
+
/**
|
|
495
|
+
* Validator-proven hostile values for one input. A header mutation touches only a field the harness
|
|
496
|
+
* owns, and an extra header proves nothing (servers tolerate them). `constrained` reports whether any
|
|
497
|
+
* transportable candidate came from a schema constraint rather than a bare type swap: a params or
|
|
498
|
+
* headers validator that accepts every type swap may simply accept any text.
|
|
499
|
+
*/
|
|
485
500
|
async function provenMutations(
|
|
486
501
|
input: ResolvedInput,
|
|
487
502
|
target: InputTarget,
|
|
488
503
|
maxMutations: number,
|
|
489
|
-
|
|
504
|
+
paramNames: readonly string[],
|
|
505
|
+
): Promise<{ readonly mutations: Mutation[]; readonly constrained: boolean }> {
|
|
490
506
|
const candidates = candidateMutations(input.value, input.reflection.jsonSchema, maxMutations)
|
|
491
507
|
const mutations: Mutation[] = []
|
|
492
508
|
const seen = new Set<string>()
|
|
509
|
+
let constrained = false
|
|
493
510
|
for (const candidate of candidates) {
|
|
511
|
+
if (target === "headers") {
|
|
512
|
+
const name = candidate.path[0]
|
|
513
|
+
if (typeof name !== "string" || !harnessOwnsHeader(name)) continue
|
|
514
|
+
if (candidate.reason === "additional-property") continue
|
|
515
|
+
}
|
|
494
516
|
let serialized: string
|
|
495
517
|
let value: unknown
|
|
496
518
|
try {
|
|
497
|
-
value = target
|
|
519
|
+
value = transported(target, candidate.value, paramNames)
|
|
498
520
|
if (value === undefined) continue
|
|
499
521
|
serialized = JSON.stringify(value)
|
|
500
522
|
} catch {
|
|
@@ -502,6 +524,7 @@ async function provenMutations(
|
|
|
502
524
|
}
|
|
503
525
|
if (seen.has(serialized)) continue
|
|
504
526
|
seen.add(serialized)
|
|
527
|
+
if (!candidate.reason.startsWith("wrong-type-")) constrained = true
|
|
505
528
|
try {
|
|
506
529
|
if (await rejectedByValidator(input.reflection, value))
|
|
507
530
|
mutations.push({ ...candidate, value })
|
|
@@ -511,7 +534,107 @@ async function provenMutations(
|
|
|
511
534
|
}
|
|
512
535
|
if (mutations.length >= maxMutations) break
|
|
513
536
|
}
|
|
514
|
-
return mutations
|
|
537
|
+
return { mutations, constrained }
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/** The header names the harness may set or mutate. */
|
|
541
|
+
const harnessOwnsHeader = (name: string): boolean => {
|
|
542
|
+
const lower = name.toLowerCase()
|
|
543
|
+
return !CREDENTIAL_HEADER.test(lower) && !RUNTIME_HEADERS.has(lower)
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
const scalarText = (value: unknown): string | undefined =>
|
|
547
|
+
typeof value === "string"
|
|
548
|
+
? value
|
|
549
|
+
: (typeof value === "number" && Number.isFinite(value)) || typeof value === "boolean"
|
|
550
|
+
? String(value)
|
|
551
|
+
: undefined
|
|
552
|
+
|
|
553
|
+
/** Path params as the router hands them over: exactly the route's names, each a non-empty segment
|
|
554
|
+
* text that survives URL normalization (no `.` or `..` part). Anything else cannot be sent. */
|
|
555
|
+
const paramsInput = (
|
|
556
|
+
value: unknown,
|
|
557
|
+
names: readonly string[],
|
|
558
|
+
): Record<string, string> | undefined => {
|
|
559
|
+
const record = recordOf(value)
|
|
560
|
+
if (record === undefined || Object.keys(record).length !== names.length) return undefined
|
|
561
|
+
const entries: [string, string][] = []
|
|
562
|
+
for (const name of names) {
|
|
563
|
+
const text = own(record, name) ? scalarText(record[name]) : undefined
|
|
564
|
+
if (
|
|
565
|
+
text === undefined ||
|
|
566
|
+
text.split("/").some((part) => part === "" || part === "." || part === "..")
|
|
567
|
+
)
|
|
568
|
+
return undefined
|
|
569
|
+
entries.push([name, text])
|
|
570
|
+
}
|
|
571
|
+
return Object.fromEntries(entries)
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/** Headers as a server reads them: lower-case names to values `Headers` carries unchanged. */
|
|
575
|
+
const headersInput = (value: unknown): Record<string, string> | undefined => {
|
|
576
|
+
const record = recordOf(value)
|
|
577
|
+
if (record === undefined) return undefined
|
|
578
|
+
const entries: [string, string][] = []
|
|
579
|
+
for (const [name, item] of Object.entries(record)) {
|
|
580
|
+
if (item === undefined) continue
|
|
581
|
+
const text = scalarText(item)
|
|
582
|
+
if (
|
|
583
|
+
text === undefined ||
|
|
584
|
+
!HEADER_NAME.test(name) ||
|
|
585
|
+
text !== text.trim() ||
|
|
586
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: a header value may carry none of these
|
|
587
|
+
/[\r\n\u0000]|[^\u0000-\u00ff]/.test(text)
|
|
588
|
+
)
|
|
589
|
+
return undefined
|
|
590
|
+
entries.push([name.toLowerCase(), text])
|
|
591
|
+
}
|
|
592
|
+
return Object.fromEntries(entries)
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/** The names a route pattern binds, each with a value its own matcher accepts. */
|
|
596
|
+
function placeholderParams(path: string): Record<string, string> {
|
|
597
|
+
let segments: readonly RoutePatternSegment[]
|
|
598
|
+
try {
|
|
599
|
+
segments = compileRoutePattern(path).segments
|
|
600
|
+
} catch {
|
|
601
|
+
return {}
|
|
602
|
+
}
|
|
603
|
+
const entries: [string, string][] = []
|
|
604
|
+
for (const segment of segments) {
|
|
605
|
+
if (segment.kind === "param") entries.push([segment.name, `${segment.name}-contract`])
|
|
606
|
+
else if (segment.kind === "wildcard") {
|
|
607
|
+
entries.push([segment.name, `${segment.name === "*" ? "path" : segment.name}/contract`])
|
|
608
|
+
} else if (segment.kind === "mixed") {
|
|
609
|
+
for (const part of segment.parts) {
|
|
610
|
+
if (part.t !== "lit") {
|
|
611
|
+
entries.push([
|
|
612
|
+
part.name,
|
|
613
|
+
part.c === undefined ? `${part.name}-contract` : satisfying(part.c),
|
|
614
|
+
])
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
return Object.fromEntries(entries)
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/** `value` when it is a record of strings, as a resolved params input always is. */
|
|
623
|
+
const stringRecordOf = (value: unknown): Readonly<Record<string, string>> | undefined => {
|
|
624
|
+
const record = recordOf(value)
|
|
625
|
+
if (record === undefined) return undefined
|
|
626
|
+
const entries = Object.entries(record)
|
|
627
|
+
return entries.every((entry): entry is [string, string] => typeof entry[1] === "string")
|
|
628
|
+
? Object.fromEntries(entries)
|
|
629
|
+
: undefined
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/** `value` in the form `target` travels in, or `undefined` when no request could carry it. */
|
|
633
|
+
function transported(target: InputTarget, value: unknown, paramNames: readonly string[]): unknown {
|
|
634
|
+
if (target === "query") return queryInput(value)
|
|
635
|
+
if (target === "headers") return headersInput(value)
|
|
636
|
+
if (target === "params") return paramsInput(value, paramNames)
|
|
637
|
+
return jsonClone(value)
|
|
515
638
|
}
|
|
516
639
|
|
|
517
640
|
const queryPairs = (value: unknown): readonly [string, string][] | undefined => {
|
|
@@ -569,11 +692,22 @@ const withReflectedJsonSchema = (
|
|
|
569
692
|
}
|
|
570
693
|
}
|
|
571
694
|
|
|
695
|
+
/** What a route's witness and its schema give one input before it is validated. */
|
|
696
|
+
interface InputSource {
|
|
697
|
+
/** The witness's own value, already in a form `transported` reads; `undefined` when it has none. */
|
|
698
|
+
readonly given: unknown
|
|
699
|
+
/** Merge a synthesized value with what the witness gave (headers and params fill gaps around it). */
|
|
700
|
+
readonly merge?: (synthesized: unknown | undefined) => unknown
|
|
701
|
+
readonly paramNames: readonly string[]
|
|
702
|
+
/** Why an opaque schema could not be read, for the gap message. */
|
|
703
|
+
readonly opaqueReason: string | undefined
|
|
704
|
+
}
|
|
705
|
+
|
|
572
706
|
async function resolveInput(
|
|
573
707
|
routeKey: string,
|
|
574
708
|
target: InputTarget,
|
|
575
709
|
reflection: SchemaReflection,
|
|
576
|
-
|
|
710
|
+
source: InputSource,
|
|
577
711
|
seed: number,
|
|
578
712
|
maxWitnessBytes: number,
|
|
579
713
|
): Promise<ResolveInput> {
|
|
@@ -590,15 +724,16 @@ async function resolveInput(
|
|
|
590
724
|
}
|
|
591
725
|
|
|
592
726
|
let value: unknown
|
|
593
|
-
if (
|
|
594
|
-
value =
|
|
727
|
+
if (source.given !== undefined && source.merge === undefined) {
|
|
728
|
+
value = source.given
|
|
595
729
|
} else if (reflection.jsonSchema !== undefined) {
|
|
596
730
|
try {
|
|
597
|
-
|
|
731
|
+
const synthesized = generateMockValue(
|
|
598
732
|
reflection.jsonSchema,
|
|
599
733
|
target,
|
|
600
734
|
seededRandom(seed ^ hash(`${routeKey}:${target}`)),
|
|
601
735
|
)
|
|
736
|
+
value = source.merge === undefined ? synthesized : source.merge(synthesized)
|
|
602
737
|
} catch (error) {
|
|
603
738
|
return {
|
|
604
739
|
ok: false,
|
|
@@ -610,6 +745,8 @@ async function resolveInput(
|
|
|
610
745
|
},
|
|
611
746
|
}
|
|
612
747
|
}
|
|
748
|
+
} else if (source.merge !== undefined && source.given !== undefined) {
|
|
749
|
+
value = source.merge(undefined)
|
|
613
750
|
} else {
|
|
614
751
|
return {
|
|
615
752
|
ok: false,
|
|
@@ -617,13 +754,13 @@ async function resolveInput(
|
|
|
617
754
|
route: routeKey,
|
|
618
755
|
target,
|
|
619
756
|
code: "NO_WITNESS",
|
|
620
|
-
message: `opaque ${target} schema needs witnesses[${JSON.stringify(routeKey)}].${target}`,
|
|
757
|
+
message: `opaque ${target} schema${source.opaqueReason === undefined ? "" : ` (${source.opaqueReason})`} needs witnesses[${JSON.stringify(routeKey)}].${target}`,
|
|
621
758
|
},
|
|
622
759
|
}
|
|
623
760
|
}
|
|
624
761
|
|
|
625
762
|
try {
|
|
626
|
-
value = target
|
|
763
|
+
value = transported(target, value, source.paramNames)
|
|
627
764
|
if (value === undefined) throw new TypeError(`${target} witness is not transport-serializable`)
|
|
628
765
|
if (valueSize(value) > maxWitnessBytes) {
|
|
629
766
|
return {
|
|
@@ -704,23 +841,48 @@ function materializePath(
|
|
|
704
841
|
return `/${filled.join("/")}`
|
|
705
842
|
}
|
|
706
843
|
|
|
844
|
+
const paramNamesOf = (laboratory: RouteLaboratory): readonly string[] =>
|
|
845
|
+
Object.keys(recordOf(laboratory.inputs.params?.value) ?? {})
|
|
846
|
+
|
|
847
|
+
/** The laboratory's resolved inputs, with `target` replaced by `value` when one is given. */
|
|
848
|
+
function inputValues(
|
|
849
|
+
laboratory: RouteLaboratory,
|
|
850
|
+
target?: InputTarget,
|
|
851
|
+
value?: unknown,
|
|
852
|
+
): Partial<Record<InputTarget, unknown>> {
|
|
853
|
+
const values: Partial<Record<InputTarget, unknown>> = {}
|
|
854
|
+
for (const name of INPUT_TARGETS) {
|
|
855
|
+
const current = name === target ? value : laboratory.inputs[name]?.value
|
|
856
|
+
if (current !== undefined) values[name] = current
|
|
857
|
+
}
|
|
858
|
+
return values
|
|
859
|
+
}
|
|
860
|
+
|
|
707
861
|
function requestFor(
|
|
708
862
|
laboratory: RouteLaboratory,
|
|
709
863
|
origin: string,
|
|
710
|
-
|
|
711
|
-
query: unknown,
|
|
864
|
+
values: Partial<Record<InputTarget, unknown>>,
|
|
712
865
|
): Request {
|
|
713
|
-
const
|
|
866
|
+
const params = stringRecordOf(values.params)
|
|
867
|
+
const path = materializePath(laboratory.route.path, params ?? laboratory.witness.params)
|
|
714
868
|
const url = new URL(path, origin)
|
|
715
|
-
for (const [key, value] of queryPairs(query) ?? []) url.searchParams.append(key, value)
|
|
869
|
+
for (const [key, value] of queryPairs(values.query) ?? []) url.searchParams.append(key, value)
|
|
716
870
|
const headers = new Headers(laboratory.witness.headers)
|
|
871
|
+
const headerValues = recordOf(values.headers)
|
|
872
|
+
if (headerValues !== undefined) {
|
|
873
|
+
// The resolved record holds the witness's headers too, so a mutation that drops one drops it here.
|
|
874
|
+
for (const name of Object.keys(recordOf(laboratory.inputs.headers?.value) ?? {})) {
|
|
875
|
+
headers.delete(name)
|
|
876
|
+
}
|
|
877
|
+
for (const [name, value] of Object.entries(headerValues)) headers.set(name, String(value))
|
|
878
|
+
}
|
|
717
879
|
const init: RequestInit = { method: laboratory.route.method, headers }
|
|
718
880
|
if (
|
|
719
881
|
laboratory.route.schema?.body !== undefined &&
|
|
720
882
|
!NO_BODY_METHODS.has(laboratory.route.method)
|
|
721
883
|
) {
|
|
722
884
|
if (!headers.has("content-type")) headers.set("content-type", "application/json")
|
|
723
|
-
init.body = JSON.stringify(body)
|
|
885
|
+
init.body = JSON.stringify(values.body)
|
|
724
886
|
}
|
|
725
887
|
return new Request(url.toString(), init)
|
|
726
888
|
}
|
|
@@ -788,10 +950,8 @@ async function executeRejection(
|
|
|
788
950
|
mutation: `${displayPath(target, mutation.path)} ${mutation.reason}`,
|
|
789
951
|
}
|
|
790
952
|
try {
|
|
791
|
-
const body = target === "body" ? mutation.value : laboratory.body?.value
|
|
792
|
-
const query = target === "query" ? mutation.value : laboratory.query?.value
|
|
793
953
|
const request = await preparedRequest(
|
|
794
|
-
requestFor(laboratory, origin,
|
|
954
|
+
requestFor(laboratory, origin, inputValues(laboratory, target, mutation.value)),
|
|
795
955
|
context,
|
|
796
956
|
options.prepareRequest,
|
|
797
957
|
)
|
|
@@ -828,17 +988,16 @@ async function shrinkFailure(
|
|
|
828
988
|
for (const value of simplerValues(current.value)) {
|
|
829
989
|
attempts += 1
|
|
830
990
|
if (attempts > maxAttempts) break
|
|
831
|
-
const reflection =
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
if (transported === undefined) continue
|
|
991
|
+
const reflection = laboratory.inputs[target]?.reflection
|
|
992
|
+
const carried = transported(target, value, paramNamesOf(laboratory))
|
|
993
|
+
if (carried === undefined) continue
|
|
835
994
|
if (reflection === undefined) continue
|
|
836
995
|
try {
|
|
837
|
-
if (!(await rejectedByValidator(reflection,
|
|
996
|
+
if (!(await rejectedByValidator(reflection, carried))) continue
|
|
838
997
|
} catch {
|
|
839
998
|
continue
|
|
840
999
|
}
|
|
841
|
-
const candidate = { ...current, value:
|
|
1000
|
+
const candidate = { ...current, value: carried }
|
|
842
1001
|
const result = await executeRejection(
|
|
843
1002
|
runtime,
|
|
844
1003
|
laboratory,
|
|
@@ -874,6 +1033,23 @@ const resultForFailure = (
|
|
|
874
1033
|
replay: { seed, caseId: base.id, runtime: base.runtime },
|
|
875
1034
|
})
|
|
876
1035
|
|
|
1036
|
+
/** A witness's headers as the lower-case record a server validates. */
|
|
1037
|
+
function headerRecordOf(init: ContractWitness["headers"]): Record<string, string> {
|
|
1038
|
+
const entries: [string, string][] = []
|
|
1039
|
+
new Headers(init).forEach((value, name) => {
|
|
1040
|
+
entries.push([name, value])
|
|
1041
|
+
})
|
|
1042
|
+
return Object.fromEntries(entries)
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
/** The `required` property names a JSON Schema object states. */
|
|
1046
|
+
const requiredNames = (schema: JsonSchema | undefined): readonly string[] => {
|
|
1047
|
+
const required = schemaRecord(schema)?.required
|
|
1048
|
+
return Array.isArray(required)
|
|
1049
|
+
? required.filter((name): name is string => typeof name === "string")
|
|
1050
|
+
: []
|
|
1051
|
+
}
|
|
1052
|
+
|
|
877
1053
|
async function buildLaboratories(
|
|
878
1054
|
routes: readonly ReflectedRoute[],
|
|
879
1055
|
options: AdversarialContractOptions,
|
|
@@ -888,95 +1064,146 @@ async function buildLaboratories(
|
|
|
888
1064
|
for (const route of routes) {
|
|
889
1065
|
const routeKey = routeKeyOf(route)
|
|
890
1066
|
const witness = options.witnesses?.[routeKey] ?? {}
|
|
891
|
-
|
|
892
|
-
|
|
1067
|
+
const schema = route.schema
|
|
1068
|
+
const witnessKey = `witnesses[${JSON.stringify(routeKey)}]`
|
|
1069
|
+
const witnessHeaders = headerRecordOf(witness.headers)
|
|
1070
|
+
const placeholders = placeholderParams(route.path)
|
|
1071
|
+
const paramNames = Object.keys(placeholders)
|
|
1072
|
+
const sendsBody = !NO_BODY_METHODS.has(route.method)
|
|
1073
|
+
const inputs: Partial<Record<InputTarget, ResolvedInput>> = {}
|
|
1074
|
+
let resolved = true
|
|
1075
|
+
|
|
1076
|
+
if (schema?.cookies !== undefined && witnessHeaders.cookie === undefined) {
|
|
1077
|
+
resolved = false
|
|
1078
|
+
gaps.push({
|
|
1079
|
+
route: routeKey,
|
|
1080
|
+
target: "cookies",
|
|
1081
|
+
code: "NO_WITNESS",
|
|
1082
|
+
message: `cookies are credentials and are never synthesized: set ${witnessKey}.headers.cookie`,
|
|
1083
|
+
})
|
|
1084
|
+
}
|
|
893
1085
|
|
|
894
|
-
|
|
1086
|
+
for (const target of INPUT_TARGETS) {
|
|
1087
|
+
const declared = schema?.[target]
|
|
1088
|
+
if (declared === undefined) continue
|
|
895
1089
|
targetCount += 1
|
|
896
|
-
if (
|
|
1090
|
+
if (target === "body" && !sendsBody) {
|
|
1091
|
+
resolved = false
|
|
897
1092
|
gaps.push({
|
|
898
1093
|
route: routeKey,
|
|
899
1094
|
target: "body",
|
|
900
1095
|
code: "UNSUPPORTED_BODY_METHOD",
|
|
901
1096
|
message: `${route.method} cannot carry a request body`,
|
|
902
1097
|
})
|
|
903
|
-
|
|
904
|
-
const resolved = await resolveInput(
|
|
905
|
-
routeKey,
|
|
906
|
-
"body",
|
|
907
|
-
withReflectedJsonSchema(
|
|
908
|
-
route.schema.body,
|
|
909
|
-
options.reflectJsonSchema ?? autoReflectJsonSchema,
|
|
910
|
-
),
|
|
911
|
-
witness,
|
|
912
|
-
seed,
|
|
913
|
-
maxWitnessBytes,
|
|
914
|
-
)
|
|
915
|
-
if (resolved.ok) body = resolved.input
|
|
916
|
-
else gaps.push(resolved.gap)
|
|
1098
|
+
continue
|
|
917
1099
|
}
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
1100
|
+
const reflection = withReflectedJsonSchema(
|
|
1101
|
+
declared,
|
|
1102
|
+
options.reflectJsonSchema ?? autoReflectJsonSchema,
|
|
1103
|
+
)
|
|
1104
|
+
const opaqueReason =
|
|
1105
|
+
reflection.jsonSchema === undefined && options.reflectJsonSchema === undefined
|
|
1106
|
+
? explainAutoReflectJsonSchema(reflection.standard)
|
|
1107
|
+
: undefined
|
|
1108
|
+
const source: InputSource =
|
|
1109
|
+
target === "params"
|
|
1110
|
+
? {
|
|
1111
|
+
given: witness.params,
|
|
1112
|
+
merge: (synthesized) => ({
|
|
1113
|
+
...placeholders,
|
|
1114
|
+
...Object.fromEntries(
|
|
1115
|
+
Object.entries(recordOf(synthesized) ?? {}).filter(([name]) =>
|
|
1116
|
+
own(placeholders, name),
|
|
1117
|
+
),
|
|
1118
|
+
),
|
|
1119
|
+
...witness.params,
|
|
1120
|
+
}),
|
|
1121
|
+
paramNames,
|
|
1122
|
+
opaqueReason,
|
|
1123
|
+
}
|
|
1124
|
+
: target === "headers"
|
|
1125
|
+
? {
|
|
1126
|
+
given: witness.headers === undefined ? undefined : witnessHeaders,
|
|
1127
|
+
merge: (synthesized) => {
|
|
1128
|
+
const merged: Record<string, unknown> = {
|
|
1129
|
+
...Object.fromEntries(
|
|
1130
|
+
Object.entries(recordOf(synthesized) ?? {}).filter(
|
|
1131
|
+
([name]) =>
|
|
1132
|
+
harnessOwnsHeader(name) && !own(witnessHeaders, name.toLowerCase()),
|
|
1133
|
+
),
|
|
1134
|
+
),
|
|
1135
|
+
...witnessHeaders,
|
|
1136
|
+
}
|
|
1137
|
+
// The request carries this for a body, so the validator must see it too.
|
|
1138
|
+
if (schema?.body !== undefined && sendsBody && !own(merged, "content-type"))
|
|
1139
|
+
merged["content-type"] = "application/json"
|
|
1140
|
+
return merged
|
|
1141
|
+
},
|
|
1142
|
+
paramNames,
|
|
1143
|
+
opaqueReason,
|
|
1144
|
+
}
|
|
1145
|
+
: {
|
|
1146
|
+
given: own(witness, target) ? witness[target] : undefined,
|
|
1147
|
+
paramNames,
|
|
1148
|
+
opaqueReason,
|
|
1149
|
+
}
|
|
1150
|
+
const result = await resolveInput(routeKey, target, reflection, source, seed, maxWitnessBytes)
|
|
1151
|
+
if (result.ok) {
|
|
1152
|
+
inputs[target] = result.input
|
|
1153
|
+
continue
|
|
1154
|
+
}
|
|
1155
|
+
resolved = false
|
|
1156
|
+
const credentials =
|
|
1157
|
+
target === "headers" && result.gap.code === "INVALID_WITNESS"
|
|
1158
|
+
? requiredNames(reflection.jsonSchema).filter(
|
|
1159
|
+
(name) => !harnessOwnsHeader(name) && !own(witnessHeaders, name.toLowerCase()),
|
|
1160
|
+
)
|
|
1161
|
+
: []
|
|
1162
|
+
gaps.push(
|
|
1163
|
+
credentials.length === 0
|
|
1164
|
+
? result.gap
|
|
1165
|
+
: {
|
|
1166
|
+
...result.gap,
|
|
1167
|
+
code: "NO_WITNESS",
|
|
1168
|
+
message: `the harness never invents credentials: set ${credentials.join(", ")} in ${witnessKey}.headers (prepareRequest may replace them per runtime)`,
|
|
1169
|
+
},
|
|
931
1170
|
)
|
|
932
|
-
if (resolved.ok) query = resolved.input
|
|
933
|
-
else gaps.push(resolved.gap)
|
|
934
1171
|
}
|
|
1172
|
+
|
|
935
1173
|
const expectedStatus = witness.expected?.status
|
|
936
1174
|
const expectedResponseSchema =
|
|
937
1175
|
expectedStatus !== undefined && (expectedStatus < 200 || expectedStatus >= 300)
|
|
938
|
-
?
|
|
939
|
-
:
|
|
1176
|
+
? schema?.errors?.[expectedStatus]
|
|
1177
|
+
: schema?.response
|
|
940
1178
|
if (options.validateResponses !== false && expectedResponseSchema !== undefined)
|
|
941
1179
|
targetCount += 1
|
|
942
1180
|
|
|
943
|
-
const
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
? await provenMutations(body, "body", maxMutations)
|
|
949
|
-
: []
|
|
950
|
-
const queryMutations =
|
|
951
|
-
query !== undefined && allInputsResolved
|
|
952
|
-
? await provenMutations(query, "query", maxMutations)
|
|
953
|
-
: []
|
|
954
|
-
if (body !== undefined && allInputsResolved && bodyMutations.length === 0) {
|
|
955
|
-
gaps.push({
|
|
956
|
-
route: routeKey,
|
|
957
|
-
target: "body",
|
|
958
|
-
code: "NO_REJECTED_MUTATION",
|
|
959
|
-
message: "the body validator accepted every generated hostile mutation",
|
|
960
|
-
})
|
|
1181
|
+
const mutations: Record<InputTarget, Mutation[]> = {
|
|
1182
|
+
headers: [],
|
|
1183
|
+
params: [],
|
|
1184
|
+
body: [],
|
|
1185
|
+
query: [],
|
|
961
1186
|
}
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
1187
|
+
for (const target of INPUT_TARGETS) {
|
|
1188
|
+
const input = inputs[target]
|
|
1189
|
+
if (input === undefined || !resolved) continue
|
|
1190
|
+
const proven = await provenMutations(input, target, maxMutations, paramNames)
|
|
1191
|
+
mutations[target] = proven.mutations
|
|
1192
|
+
// A params or headers field typed as bare text accepts every type swap, so only a constraint
|
|
1193
|
+
// the schema states is a rejection to expect there.
|
|
1194
|
+
const expectRejection = target === "body" || target === "query" || proven.constrained
|
|
1195
|
+
if (proven.mutations.length === 0 && expectRejection) {
|
|
1196
|
+
gaps.push({
|
|
1197
|
+
route: routeKey,
|
|
1198
|
+
target,
|
|
1199
|
+
code: "NO_REJECTED_MUTATION",
|
|
1200
|
+
message: `the ${target} validator accepted every generated hostile mutation`,
|
|
1201
|
+
})
|
|
1202
|
+
}
|
|
969
1203
|
}
|
|
970
|
-
laboratories.push({
|
|
971
|
-
route,
|
|
972
|
-
routeKey,
|
|
973
|
-
witness,
|
|
974
|
-
...(body === undefined ? {} : { body }),
|
|
975
|
-
...(query === undefined ? {} : { query }),
|
|
976
|
-
bodyMutations,
|
|
977
|
-
queryMutations,
|
|
978
|
-
})
|
|
1204
|
+
laboratories.push({ route, routeKey, witness, inputs, mutations, resolved })
|
|
979
1205
|
}
|
|
1206
|
+
|
|
980
1207
|
return { laboratories, targetCount }
|
|
981
1208
|
}
|
|
982
1209
|
|
|
@@ -1020,14 +1247,13 @@ export async function runAdversarialContract(
|
|
|
1020
1247
|
if (targetCount === 0) {
|
|
1021
1248
|
gaps.push({
|
|
1022
1249
|
code: "NO_CONTRACT_TARGETS",
|
|
1023
|
-
message: "selected routes declare no body, query, or response contracts",
|
|
1250
|
+
message: "selected routes declare no headers, params, body, query, or response contracts",
|
|
1024
1251
|
})
|
|
1025
1252
|
}
|
|
1026
1253
|
|
|
1027
1254
|
for (const laboratory of laboratories) {
|
|
1028
|
-
for (const target of
|
|
1029
|
-
const
|
|
1030
|
-
for (const mutation of mutations) {
|
|
1255
|
+
for (const target of INPUT_TARGETS) {
|
|
1256
|
+
for (const mutation of laboratory.mutations[target]) {
|
|
1031
1257
|
const id = mutationId(laboratory.routeKey, target, mutation)
|
|
1032
1258
|
if (only !== undefined && !only.has(id)) continue
|
|
1033
1259
|
for (const runtime of runtimes) {
|
|
@@ -1124,10 +1350,7 @@ export async function runAdversarialContract(
|
|
|
1124
1350
|
})
|
|
1125
1351
|
continue
|
|
1126
1352
|
}
|
|
1127
|
-
|
|
1128
|
-
(laboratory.route.schema?.body === undefined || laboratory.body !== undefined) &&
|
|
1129
|
-
(laboratory.route.schema?.query === undefined || laboratory.query !== undefined)
|
|
1130
|
-
if (!allInputsResolved) continue
|
|
1353
|
+
if (!laboratory.resolved) continue
|
|
1131
1354
|
|
|
1132
1355
|
for (const runtime of runtimes) {
|
|
1133
1356
|
const context: ContractCaseContext = {
|
|
@@ -1148,7 +1371,7 @@ export async function runAdversarialContract(
|
|
|
1148
1371
|
}
|
|
1149
1372
|
try {
|
|
1150
1373
|
const request = await preparedRequest(
|
|
1151
|
-
requestFor(laboratory, origin, laboratory
|
|
1374
|
+
requestFor(laboratory, origin, inputValues(laboratory)),
|
|
1152
1375
|
context,
|
|
1153
1376
|
options.prepareRequest,
|
|
1154
1377
|
)
|
|
@@ -1277,6 +1500,14 @@ export async function runAdversarialContract(
|
|
|
1277
1500
|
"contract only - declare `response:` or `errors:` on the routes you want the laboratory to verify.",
|
|
1278
1501
|
)
|
|
1279
1502
|
}
|
|
1503
|
+
// Declared on an opaque mount's word: listed so the run states what it could not test.
|
|
1504
|
+
for (const mount of reflectMounts(app)) {
|
|
1505
|
+
for (const route of mount.routes ?? []) {
|
|
1506
|
+
advisories.push(
|
|
1507
|
+
`${route.method} ${route.path} is declared by the opaque mount ${mount.path} (${mount.opaque}) and has no contract to test`,
|
|
1508
|
+
)
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1280
1511
|
return {
|
|
1281
1512
|
ok: failures.length === 0 && !fatalGap && (!coverageRequired || gaps.length === 0),
|
|
1282
1513
|
seed,
|