@nifrajs/testing 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -4
- package/dist/adversarial.d.ts +6 -38
- package/dist/adversarial.d.ts.map +1 -1
- package/dist/adversarial.js +304 -83
- package/dist/adversarial.js.map +1 -1
- package/dist/agent-eval.d.ts +25 -8
- package/dist/agent-eval.d.ts.map +1 -1
- package/dist/agent-eval.js +11 -6
- package/dist/agent-eval.js.map +1 -1
- package/dist/certification.d.ts +11 -1
- package/dist/certification.d.ts.map +1 -1
- package/dist/certification.js +136 -49
- package/dist/certification.js.map +1 -1
- package/dist/cookies.d.ts +1 -1
- package/dist/cookies.d.ts.map +1 -1
- package/dist/cookies.js +5 -3
- package/dist/cookies.js.map +1 -1
- package/dist/data-conformance.d.ts.map +1 -1
- package/dist/data-conformance.js +20 -8
- package/dist/data-conformance.js.map +1 -1
- package/dist/incident.d.ts +21 -6
- package/dist/incident.d.ts.map +1 -1
- package/dist/incident.js +99 -23
- package/dist/incident.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/trajectory.d.ts.map +1 -1
- package/dist/trajectory.js +31 -21
- package/dist/trajectory.js.map +1 -1
- package/dist/zod.d.ts +2 -2
- package/package.json +7 -7
- package/src/adversarial.ts +378 -147
- package/src/agent-eval.ts +30 -8
- package/src/certification.ts +137 -45
- package/src/cookies.ts +4 -2
- package/src/data-conformance.ts +23 -7
- package/src/incident.ts +121 -20
- package/src/index.ts +1 -0
- package/src/trajectory.ts +28 -28
- package/src/zod.ts +2 -2
package/README.md
CHANGED
|
@@ -50,6 +50,16 @@ The validator still decides which generated mutations are truly invalid; the sup
|
|
|
50
50
|
weaken validation. A missing/invalid witness is a coverage gap, not a silent pass. Set
|
|
51
51
|
`requireCoverage: false` only when gaps are intentionally advisory.
|
|
52
52
|
|
|
53
|
+
Declared `params` and `headers` get witnesses too, validated before any case runs, so a body or query
|
|
54
|
+
case is rejected by the input it mutates rather than by a placeholder param or a missing header. A `t`
|
|
55
|
+
schema synthesizes them, the witness's `params` and `headers` override, and a route param the schema
|
|
56
|
+
leaves out keeps a placeholder its pattern accepts. The harness never invents credentials:
|
|
57
|
+
`authorization`, `proxy-authorization`, `cookie`, `x-api-key`, and names ending in `-key` or containing
|
|
58
|
+
`token`, `secret` or `signature` come from the witness (and `prepareRequest`) only, and are never
|
|
59
|
+
mutated; a schema that requires one the witness lacks is a `NO_WITNESS` gap that names it. Cookies are
|
|
60
|
+
never synthesized either: a route with a `cookies` schema needs `headers.cookie` in its witness. A
|
|
61
|
+
params or header field typed as bare text accepts any text, so it is not a `NO_REJECTED_MUTATION` gap.
|
|
62
|
+
|
|
53
63
|
To exercise adapter parity, pass runtime fetch targets. The reflected `app` remains the one source of
|
|
54
64
|
the contract:
|
|
55
65
|
|
|
@@ -71,10 +81,13 @@ laboratory at production.
|
|
|
71
81
|
### Opaque validators (zod / valibot / arktype)
|
|
72
82
|
|
|
73
83
|
A `t` schema exposes JSON Schema, so witnesses and constraint-driven mutations are synthesized
|
|
74
|
-
automatically.
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
84
|
+
automatically. So does any validator that implements Standard JSON Schema (`~standard.jsonSchema`,
|
|
85
|
+
zod 4.2+ among them): the laboratory calls the schema's own converter, so nothing is resolved from disk
|
|
86
|
+
and a monorepo run from its root reads an app's zod schemas too. An older zod 4 is converted with
|
|
87
|
+
`z.toJSONSchema`, resolved from the project directory first, then from this package. A validator with
|
|
88
|
+
neither validates requests fine, but the laboratory can't inspect it - those routes report `NO_WITNESS`
|
|
89
|
+
and get only type-confusion mutations, and the gap says why when a conversion was tried and failed.
|
|
90
|
+
Pass `reflectJsonSchema` to convert any other validator, or to choose the zod bridge explicitly:
|
|
78
91
|
|
|
79
92
|
```ts
|
|
80
93
|
import { zodJsonSchema } from "@nifrajs/testing/zod"
|
package/dist/adversarial.d.ts
CHANGED
|
@@ -6,11 +6,9 @@
|
|
|
6
6
|
* Schema validator, and then drives it through one or more real fetch runtimes. Declared success
|
|
7
7
|
* responses are validated off the hot path as part of the same laboratory run.
|
|
8
8
|
*/
|
|
9
|
+
import type { ContractAuthContext, ContractCaseContext, ContractCaseKind, ContractExpectedResponse, ContractTarget, ContractWitness } from "@nifrajs/core/assurance";
|
|
9
10
|
import { type JsonSchema, type ReflectedRoute } from "@nifrajs/core/reflection";
|
|
10
|
-
type
|
|
11
|
-
export type ContractTarget = "body" | "query" | "response";
|
|
12
|
-
export type ContractCaseKind = "input-rejection" | "response-conformance";
|
|
13
|
-
export type ContractAuthContext = "authenticated" | "unauthenticated";
|
|
11
|
+
export type { ContractAuthContext, ContractCaseContext, ContractCaseKind, ContractExpectedResponse, ContractTarget, ContractWitness, };
|
|
14
12
|
/** Anything that exposes reflected routes and a Web-standard in-process fetch handler. */
|
|
15
13
|
export interface ContractTestApp {
|
|
16
14
|
routes(): readonly unknown[];
|
|
@@ -21,35 +19,6 @@ export interface ContractRuntime {
|
|
|
21
19
|
readonly name: string;
|
|
22
20
|
fetch(request: Request): Response | Promise<Response>;
|
|
23
21
|
}
|
|
24
|
-
/** A known-good request. Missing body/query values are synthesized from inspectable JSON Schema. */
|
|
25
|
-
export interface ContractWitness {
|
|
26
|
-
readonly params?: Readonly<Record<string, string>>;
|
|
27
|
-
readonly query?: unknown;
|
|
28
|
-
readonly body?: unknown;
|
|
29
|
-
readonly headers?: HeaderSource;
|
|
30
|
-
/** Explicit auth state for the prepareRequest hook; the harness never invents credentials. */
|
|
31
|
-
readonly auth?: ContractAuthContext;
|
|
32
|
-
/** Expected response for a guard/error witness. Omit for the normal success witness. */
|
|
33
|
-
readonly expected?: ContractExpectedResponse;
|
|
34
|
-
}
|
|
35
|
-
export interface ContractExpectedResponse {
|
|
36
|
-
readonly status: number;
|
|
37
|
-
readonly contentType?: "json" | "text";
|
|
38
|
-
readonly headers?: Readonly<Record<string, string>>;
|
|
39
|
-
/** When present, the decoded body is compared byte-structure-wise to this value. */
|
|
40
|
-
readonly body?: unknown;
|
|
41
|
-
}
|
|
42
|
-
/** Stable context passed to request/rejection hooks. It contains no request payloads or secrets. */
|
|
43
|
-
export interface ContractCaseContext {
|
|
44
|
-
readonly seed: number;
|
|
45
|
-
readonly caseId: string;
|
|
46
|
-
readonly route: ReflectedRoute;
|
|
47
|
-
readonly runtime: string;
|
|
48
|
-
readonly kind: ContractCaseKind;
|
|
49
|
-
readonly target: ContractTarget;
|
|
50
|
-
readonly auth?: ContractAuthContext;
|
|
51
|
-
readonly mutation?: string;
|
|
52
|
-
}
|
|
53
22
|
export interface AdversarialContractOptions {
|
|
54
23
|
/** Base origin for generated requests. Default `http://nifra.contract`. */
|
|
55
24
|
readonly origin?: string;
|
|
@@ -91,10 +60,10 @@ export interface AdversarialContractOptions {
|
|
|
91
60
|
* generation for that route. Fail-safe: throwing or returning undefined leaves the schema opaque
|
|
92
61
|
* (NO_WITNESS, as today). Ready-made for zod: `import { zodJsonSchema } from "@nifrajs/testing/zod"`.
|
|
93
62
|
*
|
|
94
|
-
* Defaults to `@nifrajs/mock`'s `autoReflectJsonSchema`, which
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
* every opaque schema to stay opaque.
|
|
63
|
+
* Defaults to `@nifrajs/mock`'s `autoReflectJsonSchema`, which reads the schema's own Standard JSON
|
|
64
|
+
* Schema converter (`~standard.jsonSchema`, zod 4.2+ and other libraries) and, for an older zod,
|
|
65
|
+
* converts via `z.toJSONSchema` - so these routes get witnesses and constraint-driven mutations with
|
|
66
|
+
* no wiring. Pass a hook to override, or `() => undefined` to force every opaque schema to stay opaque.
|
|
98
67
|
*/
|
|
99
68
|
readonly reflectJsonSchema?: (schema: unknown) => JsonSchema | undefined;
|
|
100
69
|
}
|
|
@@ -153,5 +122,4 @@ export declare class AdversarialContractError extends Error {
|
|
|
153
122
|
export declare function runAdversarialContract(app: ContractTestApp, options?: AdversarialContractOptions): Promise<AdversarialContractReport>;
|
|
154
123
|
/** Run the contract laboratory and throw an {@link AdversarialContractError} unless it is fully green. */
|
|
155
124
|
export declare function assertAdversarialContract(app: ContractTestApp, options?: AdversarialContractOptions): Promise<AdversarialContractReport>;
|
|
156
|
-
export {};
|
|
157
125
|
//# sourceMappingURL=adversarial.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adversarial.d.ts","sourceRoot":"","sources":["../src/adversarial.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;
|
|
1
|
+
{"version":3,"file":"adversarial.d.ts","sourceRoot":"","sources":["../src/adversarial.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,eAAe,EAChB,MAAM,yBAAyB,CAAA;AAMhC,OAAO,EACL,KAAK,UAAU,EACf,KAAK,cAAc,EAIpB,MAAM,0BAA0B,CAAA;AAuCjC,YAAY,EACV,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,eAAe,GAChB,CAAA;AAED,0FAA0F;AAC1F,MAAM,WAAW,eAAe;IAC9B,MAAM,IAAI,SAAS,OAAO,EAAE,CAAA;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;CACtD;AAED,mGAAmG;AACnG,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;CACtD;AAED,MAAM,WAAW,0BAA0B;IACzC,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,iEAAiE;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,iGAAiG;IACjG,QAAQ,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAA;IAC9D,2CAA2C;IAC3C,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,OAAO,CAAA;IACrD,+FAA+F;IAC/F,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,eAAe,EAAE,CAAA;IAC9C,4EAA4E;IAC5E,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAA;IACtC,wDAAwD;IACxD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,0EAA0E;IAC1E,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACvD;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,CACxB,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,mBAAmB,KACzB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC/B,iFAAiF;IACjF,QAAQ,CAAC,UAAU,CAAC,EAAE,CACpB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,mBAAmB,KACzB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC/B,4FAA4F;IAC5F,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;IACpC,kFAAkF;IAClF,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAA;IAClC,0FAA0F;IAC1F,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAA;IACjC,qDAAqD;IACrD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,sFAAsF;IACtF,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAA;IAC1C;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,UAAU,GAAG,SAAS,CAAA;CACzE;AAED,MAAM,MAAM,uBAAuB,GAC/B,qBAAqB,GACrB,cAAc,GACd,YAAY,GACZ,iBAAiB,GACjB,mBAAmB,GACnB,yBAAyB,GACzB,sBAAsB,GACtB,oBAAoB,GACpB,YAAY,GACZ,iBAAiB,GACjB,gBAAgB,CAAA;AAEpB,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAA;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAA;IACpB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,iFAAiF;IACjF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;CAChC;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS,yBAAyB,EAAE,CAAA;IACtD,QAAQ,CAAC,QAAQ,EAAE,SAAS,yBAAyB,EAAE,CAAA;IACvD,QAAQ,CAAC,IAAI,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC7C,yFAAyF;IACzF,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KACtB,CAAA;CACF;AAED,qBAAa,wBAAyB,SAAQ,KAAK;IACrC,QAAQ,CAAC,MAAM,EAAE,yBAAyB;gBAAjC,MAAM,EAAE,yBAAyB;CAoBvD;AA09BD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,GAAG,EAAE,eAAe,EACpB,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,yBAAyB,CAAC,CAqTpC;AAED,0GAA0G;AAC1G,wBAAsB,yBAAyB,CAC7C,GAAG,EAAE,eAAe,EACpB,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,yBAAyB,CAAC,CAIpC"}
|