@nifrajs/testing 4.0.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -4
- package/dist/adversarial.d.ts +6 -38
- package/dist/adversarial.d.ts.map +1 -1
- package/dist/adversarial.js +304 -83
- package/dist/adversarial.js.map +1 -1
- package/dist/agent-eval.d.ts +25 -8
- package/dist/agent-eval.d.ts.map +1 -1
- package/dist/agent-eval.js +11 -6
- package/dist/agent-eval.js.map +1 -1
- package/dist/certification.d.ts +11 -1
- package/dist/certification.d.ts.map +1 -1
- package/dist/certification.js +136 -49
- package/dist/certification.js.map +1 -1
- package/dist/cookies.d.ts +1 -1
- package/dist/cookies.d.ts.map +1 -1
- package/dist/cookies.js +5 -3
- package/dist/cookies.js.map +1 -1
- package/dist/data-conformance.d.ts.map +1 -1
- package/dist/data-conformance.js +20 -8
- package/dist/data-conformance.js.map +1 -1
- package/dist/incident.d.ts +21 -6
- package/dist/incident.d.ts.map +1 -1
- package/dist/incident.js +99 -23
- package/dist/incident.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/trajectory.d.ts.map +1 -1
- package/dist/trajectory.js +31 -21
- package/dist/trajectory.js.map +1 -1
- package/dist/zod.d.ts +2 -2
- package/package.json +7 -7
- package/src/adversarial.ts +378 -147
- package/src/agent-eval.ts +30 -8
- package/src/certification.ts +137 -45
- package/src/cookies.ts +4 -2
- package/src/data-conformance.ts +23 -7
- package/src/incident.ts +121 -20
- package/src/index.ts +1 -0
- package/src/trajectory.ts +28 -28
- package/src/zod.ts +2 -2
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nifra Authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -50,6 +50,16 @@ The validator still decides which generated mutations are truly invalid; the sup
|
|
|
50
50
|
weaken validation. A missing/invalid witness is a coverage gap, not a silent pass. Set
|
|
51
51
|
`requireCoverage: false` only when gaps are intentionally advisory.
|
|
52
52
|
|
|
53
|
+
Declared `params` and `headers` get witnesses too, validated before any case runs, so a body or query
|
|
54
|
+
case is rejected by the input it mutates rather than by a placeholder param or a missing header. A `t`
|
|
55
|
+
schema synthesizes them, the witness's `params` and `headers` override, and a route param the schema
|
|
56
|
+
leaves out keeps a placeholder its pattern accepts. The harness never invents credentials:
|
|
57
|
+
`authorization`, `proxy-authorization`, `cookie`, `x-api-key`, and names ending in `-key` or containing
|
|
58
|
+
`token`, `secret` or `signature` come from the witness (and `prepareRequest`) only, and are never
|
|
59
|
+
mutated; a schema that requires one the witness lacks is a `NO_WITNESS` gap that names it. Cookies are
|
|
60
|
+
never synthesized either: a route with a `cookies` schema needs `headers.cookie` in its witness. A
|
|
61
|
+
params or header field typed as bare text accepts any text, so it is not a `NO_REJECTED_MUTATION` gap.
|
|
62
|
+
|
|
53
63
|
To exercise adapter parity, pass runtime fetch targets. The reflected `app` remains the one source of
|
|
54
64
|
the contract:
|
|
55
65
|
|
|
@@ -71,10 +81,13 @@ laboratory at production.
|
|
|
71
81
|
### Opaque validators (zod / valibot / arktype)
|
|
72
82
|
|
|
73
83
|
A `t` schema exposes JSON Schema, so witnesses and constraint-driven mutations are synthesized
|
|
74
|
-
automatically.
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
84
|
+
automatically. So does any validator that implements Standard JSON Schema (`~standard.jsonSchema`,
|
|
85
|
+
zod 4.2+ among them): the laboratory calls the schema's own converter, so nothing is resolved from disk
|
|
86
|
+
and a monorepo run from its root reads an app's zod schemas too. An older zod 4 is converted with
|
|
87
|
+
`z.toJSONSchema`, resolved from the project directory first, then from this package. A validator with
|
|
88
|
+
neither validates requests fine, but the laboratory can't inspect it - those routes report `NO_WITNESS`
|
|
89
|
+
and get only type-confusion mutations, and the gap says why when a conversion was tried and failed.
|
|
90
|
+
Pass `reflectJsonSchema` to convert any other validator, or to choose the zod bridge explicitly:
|
|
78
91
|
|
|
79
92
|
```ts
|
|
80
93
|
import { zodJsonSchema } from "@nifrajs/testing/zod"
|
package/dist/adversarial.d.ts
CHANGED
|
@@ -6,11 +6,9 @@
|
|
|
6
6
|
* Schema validator, and then drives it through one or more real fetch runtimes. Declared success
|
|
7
7
|
* responses are validated off the hot path as part of the same laboratory run.
|
|
8
8
|
*/
|
|
9
|
+
import type { ContractAuthContext, ContractCaseContext, ContractCaseKind, ContractExpectedResponse, ContractTarget, ContractWitness } from "@nifrajs/core/assurance";
|
|
9
10
|
import { type JsonSchema, type ReflectedRoute } from "@nifrajs/core/reflection";
|
|
10
|
-
type
|
|
11
|
-
export type ContractTarget = "body" | "query" | "response";
|
|
12
|
-
export type ContractCaseKind = "input-rejection" | "response-conformance";
|
|
13
|
-
export type ContractAuthContext = "authenticated" | "unauthenticated";
|
|
11
|
+
export type { ContractAuthContext, ContractCaseContext, ContractCaseKind, ContractExpectedResponse, ContractTarget, ContractWitness, };
|
|
14
12
|
/** Anything that exposes reflected routes and a Web-standard in-process fetch handler. */
|
|
15
13
|
export interface ContractTestApp {
|
|
16
14
|
routes(): readonly unknown[];
|
|
@@ -21,35 +19,6 @@ export interface ContractRuntime {
|
|
|
21
19
|
readonly name: string;
|
|
22
20
|
fetch(request: Request): Response | Promise<Response>;
|
|
23
21
|
}
|
|
24
|
-
/** A known-good request. Missing body/query values are synthesized from inspectable JSON Schema. */
|
|
25
|
-
export interface ContractWitness {
|
|
26
|
-
readonly params?: Readonly<Record<string, string>>;
|
|
27
|
-
readonly query?: unknown;
|
|
28
|
-
readonly body?: unknown;
|
|
29
|
-
readonly headers?: HeaderSource;
|
|
30
|
-
/** Explicit auth state for the prepareRequest hook; the harness never invents credentials. */
|
|
31
|
-
readonly auth?: ContractAuthContext;
|
|
32
|
-
/** Expected response for a guard/error witness. Omit for the normal success witness. */
|
|
33
|
-
readonly expected?: ContractExpectedResponse;
|
|
34
|
-
}
|
|
35
|
-
export interface ContractExpectedResponse {
|
|
36
|
-
readonly status: number;
|
|
37
|
-
readonly contentType?: "json" | "text";
|
|
38
|
-
readonly headers?: Readonly<Record<string, string>>;
|
|
39
|
-
/** When present, the decoded body is compared byte-structure-wise to this value. */
|
|
40
|
-
readonly body?: unknown;
|
|
41
|
-
}
|
|
42
|
-
/** Stable context passed to request/rejection hooks. It contains no request payloads or secrets. */
|
|
43
|
-
export interface ContractCaseContext {
|
|
44
|
-
readonly seed: number;
|
|
45
|
-
readonly caseId: string;
|
|
46
|
-
readonly route: ReflectedRoute;
|
|
47
|
-
readonly runtime: string;
|
|
48
|
-
readonly kind: ContractCaseKind;
|
|
49
|
-
readonly target: ContractTarget;
|
|
50
|
-
readonly auth?: ContractAuthContext;
|
|
51
|
-
readonly mutation?: string;
|
|
52
|
-
}
|
|
53
22
|
export interface AdversarialContractOptions {
|
|
54
23
|
/** Base origin for generated requests. Default `http://nifra.contract`. */
|
|
55
24
|
readonly origin?: string;
|
|
@@ -91,10 +60,10 @@ export interface AdversarialContractOptions {
|
|
|
91
60
|
* generation for that route. Fail-safe: throwing or returning undefined leaves the schema opaque
|
|
92
61
|
* (NO_WITNESS, as today). Ready-made for zod: `import { zodJsonSchema } from "@nifrajs/testing/zod"`.
|
|
93
62
|
*
|
|
94
|
-
* Defaults to `@nifrajs/mock`'s `autoReflectJsonSchema`, which
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
* every opaque schema to stay opaque.
|
|
63
|
+
* Defaults to `@nifrajs/mock`'s `autoReflectJsonSchema`, which reads the schema's own Standard JSON
|
|
64
|
+
* Schema converter (`~standard.jsonSchema`, zod 4.2+ and other libraries) and, for an older zod,
|
|
65
|
+
* converts via `z.toJSONSchema` - so these routes get witnesses and constraint-driven mutations with
|
|
66
|
+
* no wiring. Pass a hook to override, or `() => undefined` to force every opaque schema to stay opaque.
|
|
98
67
|
*/
|
|
99
68
|
readonly reflectJsonSchema?: (schema: unknown) => JsonSchema | undefined;
|
|
100
69
|
}
|
|
@@ -153,5 +122,4 @@ export declare class AdversarialContractError extends Error {
|
|
|
153
122
|
export declare function runAdversarialContract(app: ContractTestApp, options?: AdversarialContractOptions): Promise<AdversarialContractReport>;
|
|
154
123
|
/** Run the contract laboratory and throw an {@link AdversarialContractError} unless it is fully green. */
|
|
155
124
|
export declare function assertAdversarialContract(app: ContractTestApp, options?: AdversarialContractOptions): Promise<AdversarialContractReport>;
|
|
156
|
-
export {};
|
|
157
125
|
//# sourceMappingURL=adversarial.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adversarial.d.ts","sourceRoot":"","sources":["../src/adversarial.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;
|
|
1
|
+
{"version":3,"file":"adversarial.d.ts","sourceRoot":"","sources":["../src/adversarial.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,eAAe,EAChB,MAAM,yBAAyB,CAAA;AAMhC,OAAO,EACL,KAAK,UAAU,EACf,KAAK,cAAc,EAIpB,MAAM,0BAA0B,CAAA;AAuCjC,YAAY,EACV,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,eAAe,GAChB,CAAA;AAED,0FAA0F;AAC1F,MAAM,WAAW,eAAe;IAC9B,MAAM,IAAI,SAAS,OAAO,EAAE,CAAA;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;CACtD;AAED,mGAAmG;AACnG,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;CACtD;AAED,MAAM,WAAW,0BAA0B;IACzC,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,iEAAiE;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,iGAAiG;IACjG,QAAQ,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAA;IAC9D,2CAA2C;IAC3C,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,OAAO,CAAA;IACrD,+FAA+F;IAC/F,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,eAAe,EAAE,CAAA;IAC9C,4EAA4E;IAC5E,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAA;IACtC,wDAAwD;IACxD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,0EAA0E;IAC1E,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACvD;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,CACxB,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,mBAAmB,KACzB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC/B,iFAAiF;IACjF,QAAQ,CAAC,UAAU,CAAC,EAAE,CACpB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,mBAAmB,KACzB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC/B,4FAA4F;IAC5F,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;IACpC,kFAAkF;IAClF,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAA;IAClC,0FAA0F;IAC1F,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAA;IACjC,qDAAqD;IACrD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,sFAAsF;IACtF,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAA;IAC1C;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,UAAU,GAAG,SAAS,CAAA;CACzE;AAED,MAAM,MAAM,uBAAuB,GAC/B,qBAAqB,GACrB,cAAc,GACd,YAAY,GACZ,iBAAiB,GACjB,mBAAmB,GACnB,yBAAyB,GACzB,sBAAsB,GACtB,oBAAoB,GACpB,YAAY,GACZ,iBAAiB,GACjB,gBAAgB,CAAA;AAEpB,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAA;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAA;IACpB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,iFAAiF;IACjF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;CAChC;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS,yBAAyB,EAAE,CAAA;IACtD,QAAQ,CAAC,QAAQ,EAAE,SAAS,yBAAyB,EAAE,CAAA;IACvD,QAAQ,CAAC,IAAI,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC7C,yFAAyF;IACzF,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KACtB,CAAA;CACF;AAED,qBAAa,wBAAyB,SAAQ,KAAK;IACrC,QAAQ,CAAC,MAAM,EAAE,yBAAyB;gBAAjC,MAAM,EAAE,yBAAyB;CAoBvD;AA09BD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,GAAG,EAAE,eAAe,EACpB,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,yBAAyB,CAAC,CAqTpC;AAED,0GAA0G;AAC1G,wBAAsB,yBAAyB,CAC7C,GAAG,EAAE,eAAe,EACpB,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,yBAAyB,CAAC,CAIpC"}
|