@nifrajs/testing 3.3.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/adversarial.d.ts +14 -1
- package/dist/adversarial.d.ts.map +1 -1
- package/dist/adversarial.js +113 -25
- package/dist/adversarial.js.map +1 -1
- package/dist/agent-eval.d.ts.map +1 -1
- package/dist/agent-eval.js +3 -1
- package/dist/agent-eval.js.map +1 -1
- package/dist/certification.d.ts +28 -1
- package/dist/certification.d.ts.map +1 -1
- package/dist/certification.js +119 -51
- package/dist/certification.js.map +1 -1
- package/dist/contract-lab.js +1 -1
- package/dist/contract-lab.js.map +1 -1
- package/dist/data-conformance.d.ts +72 -0
- package/dist/data-conformance.d.ts.map +1 -0
- package/dist/data-conformance.js +198 -0
- package/dist/data-conformance.js.map +1 -0
- package/dist/e2e.d.ts +63 -0
- package/dist/e2e.d.ts.map +1 -0
- package/dist/e2e.js +56 -0
- package/dist/e2e.js.map +1 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/prediction-lab.d.ts +65 -0
- package/dist/prediction-lab.d.ts.map +1 -0
- package/dist/prediction-lab.js +530 -0
- package/dist/prediction-lab.js.map +1 -0
- package/package.json +20 -7
- package/src/adversarial.ts +144 -28
- package/src/agent-eval.ts +4 -3
- package/src/certification.ts +138 -45
- package/src/contract-lab.ts +1 -1
- package/src/data-conformance.ts +297 -0
- package/src/e2e.ts +112 -0
- package/src/index.ts +28 -0
- package/src/prediction-lab.ts +692 -0
package/src/adversarial.ts
CHANGED
|
@@ -7,6 +7,11 @@
|
|
|
7
7
|
* responses are validated off the hot path as part of the same laboratory run.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import {
|
|
11
|
+
compileRoutePattern,
|
|
12
|
+
type ParamConstraint,
|
|
13
|
+
type RoutePatternSegment,
|
|
14
|
+
} from "@nifrajs/core/pattern"
|
|
10
15
|
import {
|
|
11
16
|
type JsonSchema,
|
|
12
17
|
type ReflectedRoute,
|
|
@@ -26,6 +31,7 @@ type HeaderSource = ConstructorParameters<typeof Headers>[0]
|
|
|
26
31
|
type InputTarget = "body" | "query"
|
|
27
32
|
export type ContractTarget = "body" | "query" | "response"
|
|
28
33
|
export type ContractCaseKind = "input-rejection" | "response-conformance"
|
|
34
|
+
export type ContractAuthContext = "authenticated" | "unauthenticated"
|
|
29
35
|
|
|
30
36
|
/** Anything that exposes reflected routes and a Web-standard in-process fetch handler. */
|
|
31
37
|
export interface ContractTestApp {
|
|
@@ -45,6 +51,18 @@ export interface ContractWitness {
|
|
|
45
51
|
readonly query?: unknown
|
|
46
52
|
readonly body?: unknown
|
|
47
53
|
readonly headers?: HeaderSource
|
|
54
|
+
/** Explicit auth state for the prepareRequest hook; the harness never invents credentials. */
|
|
55
|
+
readonly auth?: ContractAuthContext
|
|
56
|
+
/** Expected response for a guard/error witness. Omit for the normal success witness. */
|
|
57
|
+
readonly expected?: ContractExpectedResponse
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface ContractExpectedResponse {
|
|
61
|
+
readonly status: number
|
|
62
|
+
readonly contentType?: "json" | "text"
|
|
63
|
+
readonly headers?: Readonly<Record<string, string>>
|
|
64
|
+
/** When present, the decoded body is compared byte-structure-wise to this value. */
|
|
65
|
+
readonly body?: unknown
|
|
48
66
|
}
|
|
49
67
|
|
|
50
68
|
/** Stable context passed to request/rejection hooks. It contains no request payloads or secrets. */
|
|
@@ -55,6 +73,7 @@ export interface ContractCaseContext {
|
|
|
55
73
|
readonly runtime: string
|
|
56
74
|
readonly kind: ContractCaseKind
|
|
57
75
|
readonly target: ContractTarget
|
|
76
|
+
readonly auth?: ContractAuthContext
|
|
58
77
|
readonly mutation?: string
|
|
59
78
|
}
|
|
60
79
|
|
|
@@ -121,6 +140,7 @@ export type ContractCoverageGapCode =
|
|
|
121
140
|
| "WITNESS_TOO_LARGE"
|
|
122
141
|
| "UNSUPPORTED_BODY_METHOD"
|
|
123
142
|
| "NO_REJECTED_MUTATION"
|
|
143
|
+
| "NO_STATUS_CONTRACT"
|
|
124
144
|
| "NO_RUNTIME"
|
|
125
145
|
| "INVALID_RUNTIME"
|
|
126
146
|
| "CASE_NOT_FOUND"
|
|
@@ -257,6 +277,19 @@ const jsonClone = (value: unknown): unknown => {
|
|
|
257
277
|
return JSON.parse(serialized) as unknown
|
|
258
278
|
}
|
|
259
279
|
|
|
280
|
+
const canonical = (value: unknown): string => {
|
|
281
|
+
if (value === undefined) return "undefined"
|
|
282
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`
|
|
283
|
+
if (value !== null && typeof value === "object") {
|
|
284
|
+
const record = value as Readonly<Record<string, unknown>>
|
|
285
|
+
return `{${Object.keys(record)
|
|
286
|
+
.sort()
|
|
287
|
+
.map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`)
|
|
288
|
+
.join(",")}}`
|
|
289
|
+
}
|
|
290
|
+
return JSON.stringify(value)
|
|
291
|
+
}
|
|
292
|
+
|
|
260
293
|
const valueSize = (value: unknown): number => new TextEncoder().encode(JSON.stringify(value)).length
|
|
261
294
|
|
|
262
295
|
const displayPath = (target: InputTarget, path: readonly (string | number)[]): string =>
|
|
@@ -268,7 +301,14 @@ const displayPath = (target: InputTarget, path: readonly (string | number)[]): s
|
|
|
268
301
|
const mutationId = (routeKey: string, target: InputTarget, mutation: Mutation): string =>
|
|
269
302
|
`${routeKey} :: ${displayPath(target, mutation.path)} :: ${mutation.reason}`
|
|
270
303
|
|
|
271
|
-
const responseId = (routeKey: string): string =>
|
|
304
|
+
const responseId = (routeKey: string, witness: ContractWitness): string => {
|
|
305
|
+
if (witness.auth === undefined && witness.expected === undefined) {
|
|
306
|
+
return `${routeKey} :: response-conformance`
|
|
307
|
+
}
|
|
308
|
+
const auth = witness.auth ?? "default"
|
|
309
|
+
const status = witness.expected?.status ?? "success"
|
|
310
|
+
return `${routeKey} :: ${auth} :: ${status} :: response-conformance`
|
|
311
|
+
}
|
|
272
312
|
|
|
273
313
|
const typeKind = (value: unknown): string => {
|
|
274
314
|
if (value === null) return "null"
|
|
@@ -621,25 +661,48 @@ async function resolveInput(
|
|
|
621
661
|
}
|
|
622
662
|
}
|
|
623
663
|
|
|
624
|
-
|
|
664
|
+
/**
|
|
665
|
+
* A value `constraint` accepts: its first listed value, or one accepted character repeated to the
|
|
666
|
+
* shortest accepted length. Every character a constraint can name is legal in a path as written.
|
|
667
|
+
*/
|
|
668
|
+
function satisfying(constraint: ParamConstraint): string {
|
|
669
|
+
if (constraint.oneOf !== undefined) return constraint.oneOf[0]!
|
|
670
|
+
// Digits and letters first, so the usual classes give a value that reads like one.
|
|
671
|
+
let code = 48
|
|
672
|
+
while (constraint.mask[code] !== "1") code = code === 127 ? 33 : code + 1
|
|
673
|
+
return String.fromCharCode(code).repeat(constraint.min)
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
/** Params filled from the witness or a placeholder the route accepts, split by the router's compiler. */
|
|
677
|
+
function materializePath(
|
|
625
678
|
path: string,
|
|
626
679
|
params: Readonly<Record<string, string>> | undefined,
|
|
627
|
-
): string
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
.
|
|
680
|
+
): string {
|
|
681
|
+
let segments: readonly RoutePatternSegment[]
|
|
682
|
+
try {
|
|
683
|
+
segments = compileRoutePattern(path).segments
|
|
684
|
+
} catch {
|
|
685
|
+
return path
|
|
686
|
+
}
|
|
687
|
+
const fill = (name: string, constraint?: ParamConstraint): string => {
|
|
688
|
+
const given = params?.[name]
|
|
689
|
+
if (given !== undefined) return encodeURIComponent(given)
|
|
690
|
+
return constraint === undefined ? `${name}-contract` : satisfying(constraint)
|
|
691
|
+
}
|
|
692
|
+
const filled = segments.map((segment) => {
|
|
693
|
+
if (segment.kind === "static") return segment.value
|
|
694
|
+
if (segment.kind === "param") return fill(segment.name)
|
|
695
|
+
if (segment.kind === "mixed") {
|
|
696
|
+
return segment.parts
|
|
697
|
+
.map((part) => (part.t === "lit" ? part.v : fill(part.name, part.c)))
|
|
698
|
+
.join("")
|
|
699
|
+
}
|
|
700
|
+
const value =
|
|
701
|
+
params?.[segment.name] ?? `${segment.name === "*" ? "path" : segment.name}/contract`
|
|
702
|
+
return value.split("/").filter(Boolean).map(encodeURIComponent).join("/")
|
|
703
|
+
})
|
|
704
|
+
return `/${filled.join("/")}`
|
|
705
|
+
}
|
|
643
706
|
|
|
644
707
|
function requestFor(
|
|
645
708
|
laboratory: RouteLaboratory,
|
|
@@ -721,6 +784,7 @@ async function executeRejection(
|
|
|
721
784
|
runtime: runtime.name,
|
|
722
785
|
kind: "input-rejection",
|
|
723
786
|
target,
|
|
787
|
+
...(laboratory.witness.auth === undefined ? {} : { auth: laboratory.witness.auth }),
|
|
724
788
|
mutation: `${displayPath(target, mutation.path)} ${mutation.reason}`,
|
|
725
789
|
}
|
|
726
790
|
try {
|
|
@@ -868,7 +932,12 @@ async function buildLaboratories(
|
|
|
868
932
|
if (resolved.ok) query = resolved.input
|
|
869
933
|
else gaps.push(resolved.gap)
|
|
870
934
|
}
|
|
871
|
-
|
|
935
|
+
const expectedStatus = witness.expected?.status
|
|
936
|
+
const expectedResponseSchema =
|
|
937
|
+
expectedStatus !== undefined && (expectedStatus < 200 || expectedStatus >= 300)
|
|
938
|
+
? route.schema?.errors?.[expectedStatus]
|
|
939
|
+
: route.schema?.response
|
|
940
|
+
if (options.validateResponses !== false && expectedResponseSchema !== undefined)
|
|
872
941
|
targetCount += 1
|
|
873
942
|
|
|
874
943
|
const allInputsResolved =
|
|
@@ -1027,10 +1096,25 @@ export async function runAdversarialContract(
|
|
|
1027
1096
|
}
|
|
1028
1097
|
}
|
|
1029
1098
|
|
|
1030
|
-
const
|
|
1031
|
-
|
|
1032
|
-
|
|
1099
|
+
const expected = laboratory.witness.expected
|
|
1100
|
+
const responseSchema =
|
|
1101
|
+
expected !== undefined && (expected.status < 200 || expected.status >= 300)
|
|
1102
|
+
? laboratory.route.schema?.errors?.[expected.status]
|
|
1103
|
+
: laboratory.route.schema?.response
|
|
1104
|
+
if (options.validateResponses === false) continue
|
|
1105
|
+
const id = responseId(laboratory.routeKey, laboratory.witness)
|
|
1033
1106
|
if (only !== undefined && !only.has(id)) continue
|
|
1107
|
+
if (responseSchema === undefined) {
|
|
1108
|
+
if (expected !== undefined && (expected.status < 200 || expected.status >= 300)) {
|
|
1109
|
+
gaps.push({
|
|
1110
|
+
route: laboratory.routeKey,
|
|
1111
|
+
target: "response",
|
|
1112
|
+
code: "NO_STATUS_CONTRACT",
|
|
1113
|
+
message: `expected status ${expected.status} has no declared error response contract`,
|
|
1114
|
+
})
|
|
1115
|
+
}
|
|
1116
|
+
continue
|
|
1117
|
+
}
|
|
1034
1118
|
if (responseSchema.standard === undefined) {
|
|
1035
1119
|
gaps.push({
|
|
1036
1120
|
route: laboratory.routeKey,
|
|
@@ -1053,6 +1137,7 @@ export async function runAdversarialContract(
|
|
|
1053
1137
|
runtime: runtime.name,
|
|
1054
1138
|
kind: "response-conformance",
|
|
1055
1139
|
target: "response",
|
|
1140
|
+
...(laboratory.witness.auth === undefined ? {} : { auth: laboratory.witness.auth }),
|
|
1056
1141
|
}
|
|
1057
1142
|
const base = {
|
|
1058
1143
|
id,
|
|
@@ -1069,13 +1154,15 @@ export async function runAdversarialContract(
|
|
|
1069
1154
|
)
|
|
1070
1155
|
const response = await runtime.fetch(request)
|
|
1071
1156
|
const status = response.status
|
|
1072
|
-
if (!response.ok) {
|
|
1157
|
+
if (expected === undefined ? !response.ok : response.status !== expected.status) {
|
|
1073
1158
|
await discardResponse(response)
|
|
1074
1159
|
results.push(
|
|
1075
1160
|
resultForFailure(
|
|
1076
1161
|
{ ...base, status },
|
|
1077
1162
|
seed,
|
|
1078
|
-
|
|
1163
|
+
expected === undefined
|
|
1164
|
+
? `valid witness returned non-success status ${status}`
|
|
1165
|
+
: `expected status ${expected.status}, received ${status}`,
|
|
1079
1166
|
),
|
|
1080
1167
|
)
|
|
1081
1168
|
continue
|
|
@@ -1083,7 +1170,12 @@ export async function runAdversarialContract(
|
|
|
1083
1170
|
const text = await response.text()
|
|
1084
1171
|
let value: unknown
|
|
1085
1172
|
try {
|
|
1086
|
-
value =
|
|
1173
|
+
value =
|
|
1174
|
+
expected?.contentType === "text"
|
|
1175
|
+
? text
|
|
1176
|
+
: text.length === 0
|
|
1177
|
+
? undefined
|
|
1178
|
+
: (JSON.parse(text) as unknown)
|
|
1087
1179
|
} catch {
|
|
1088
1180
|
results.push(
|
|
1089
1181
|
resultForFailure(
|
|
@@ -1094,6 +1186,26 @@ export async function runAdversarialContract(
|
|
|
1094
1186
|
)
|
|
1095
1187
|
continue
|
|
1096
1188
|
}
|
|
1189
|
+
if (expected !== undefined) {
|
|
1190
|
+
const expectedContentType =
|
|
1191
|
+
expected.contentType === "text" ? "text/plain" : "application/json"
|
|
1192
|
+
const actualContentType = response.headers.get("content-type")?.split(";", 1)[0] ?? ""
|
|
1193
|
+
const headersMatch = Object.entries(expected.headers ?? {}).every(
|
|
1194
|
+
([name, headerValue]) => response.headers.get(name) === headerValue,
|
|
1195
|
+
)
|
|
1196
|
+
const bodyMatch =
|
|
1197
|
+
!Object.hasOwn(expected, "body") || canonical(value) === canonical(expected.body)
|
|
1198
|
+
if (actualContentType !== expectedContentType || !headersMatch || !bodyMatch) {
|
|
1199
|
+
results.push(
|
|
1200
|
+
resultForFailure(
|
|
1201
|
+
{ ...base, status },
|
|
1202
|
+
seed,
|
|
1203
|
+
`expected response envelope ${expectedContentType} ${canonical(expected.body)}, received ${actualContentType || "(missing content type)"} ${canonical(value)}`,
|
|
1204
|
+
),
|
|
1205
|
+
)
|
|
1206
|
+
continue
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1097
1209
|
const validation = await responseSchema.standard["~standard"].validate(value)
|
|
1098
1210
|
if (validation.issues !== undefined) {
|
|
1099
1211
|
const first = validation.issues[0]
|
|
@@ -1153,12 +1265,16 @@ export async function runAdversarialContract(
|
|
|
1153
1265
|
if (
|
|
1154
1266
|
options.validateResponses !== false &&
|
|
1155
1267
|
routes.length > 0 &&
|
|
1156
|
-
!laboratories.some(
|
|
1268
|
+
!laboratories.some(
|
|
1269
|
+
(laboratory) =>
|
|
1270
|
+
laboratory.route.schema?.response !== undefined ||
|
|
1271
|
+
Object.keys(laboratory.route.schema?.errors ?? {}).length > 0,
|
|
1272
|
+
)
|
|
1157
1273
|
) {
|
|
1158
1274
|
advisories.push(
|
|
1159
|
-
"validateResponses is on, but no selected route declares a `response` schema, so response " +
|
|
1275
|
+
"validateResponses is on, but no selected route declares a `response` or status-specific `errors` schema, so response " +
|
|
1160
1276
|
"conformance checked 0 targets. Response types inferred from a handler's return are a client-side " +
|
|
1161
|
-
"contract only - declare `response:` on the routes you want the laboratory to verify.",
|
|
1277
|
+
"contract only - declare `response:` or `errors:` on the routes you want the laboratory to verify.",
|
|
1162
1278
|
)
|
|
1163
1279
|
}
|
|
1164
1280
|
return {
|
package/src/agent-eval.ts
CHANGED
|
@@ -17,6 +17,9 @@ import { type FailureDirective, FailureInjectedError, runFailureScenario } from
|
|
|
17
17
|
import { type FaultProfile, runFaultProfile } from "./fault-profile.ts"
|
|
18
18
|
import { type EffectLedger, type IdempotencyProof, proveIdempotency } from "./idempotency.ts"
|
|
19
19
|
|
|
20
|
+
/** Code-unit order, the same in every locale: a case digest must not depend on the machine scoring it. */
|
|
21
|
+
const byCodeUnit = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0)
|
|
22
|
+
|
|
20
23
|
const TOKEN = /^[a-z0-9][a-z0-9._:-]{0,127}$/
|
|
21
24
|
const MAX_CASES = 512
|
|
22
25
|
const MAX_RUBRICS = 32
|
|
@@ -509,9 +512,7 @@ export function defineAgentEvalSuite(spec: AgentEvalSuiteSpec): AgentEvalSuite {
|
|
|
509
512
|
throw new TypeError(`agent eval: case '${caseId}' scored rubric '${rubric.id}' twice`)
|
|
510
513
|
perRubric.set(rubric.id, parseRubricVerdict(rubric, raw))
|
|
511
514
|
}
|
|
512
|
-
const verdicts = [...perRubric.values()].sort((a, b) =>
|
|
513
|
-
a.rubricId.localeCompare(b.rubricId),
|
|
514
|
-
)
|
|
515
|
+
const verdicts = [...perRubric.values()].sort((a, b) => byCodeUnit(a.rubricId, b.rubricId))
|
|
515
516
|
cases.push({
|
|
516
517
|
caseId,
|
|
517
518
|
verdicts: Object.freeze(verdicts),
|
package/src/certification.ts
CHANGED
|
@@ -8,9 +8,24 @@ export interface CertificationContext {
|
|
|
8
8
|
readonly key: (suffix: string) => string
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
+
/** Portable identity for the artifact, runtime, and witness that produced one check result. */
|
|
12
|
+
export interface CertificationTarget {
|
|
13
|
+
/** Published adapter/package name, for example `@nifrajs/node`. */
|
|
14
|
+
readonly adapter?: string
|
|
15
|
+
/** Runtime that executed the witness, for example `node`, `deno`, or `workerd`. */
|
|
16
|
+
readonly runtime?: string
|
|
17
|
+
/** Checked-in or built artifact path, when the witness exercises one. */
|
|
18
|
+
readonly artifact?: string
|
|
19
|
+
/** Source path that owns the behavior, when it differs from the artifact. */
|
|
20
|
+
readonly source?: string
|
|
21
|
+
/** Stable category of witness, useful when one report mixes several targets. */
|
|
22
|
+
readonly witnessKind?: string
|
|
23
|
+
}
|
|
24
|
+
|
|
11
25
|
export interface CertificationCheck<Adapter> {
|
|
12
26
|
readonly id: string
|
|
13
27
|
readonly capability: string
|
|
28
|
+
readonly target?: CertificationTarget
|
|
14
29
|
readonly run: (adapter: Adapter, context: CertificationContext) => void | Promise<void>
|
|
15
30
|
}
|
|
16
31
|
|
|
@@ -25,6 +40,7 @@ export interface CertificationCheckEvidence {
|
|
|
25
40
|
readonly id: string
|
|
26
41
|
readonly capability: string
|
|
27
42
|
readonly ok: boolean
|
|
43
|
+
readonly target?: CertificationTarget
|
|
28
44
|
/** Error class only. Messages may contain credentials/provider payloads and are never evidence. */
|
|
29
45
|
readonly errorName?: string
|
|
30
46
|
}
|
|
@@ -40,6 +56,7 @@ export interface AdapterCertificationReport {
|
|
|
40
56
|
readonly ok: boolean
|
|
41
57
|
readonly profile: { readonly id: string; readonly version: number }
|
|
42
58
|
readonly adapterId: string
|
|
59
|
+
readonly target?: CertificationTarget
|
|
43
60
|
readonly checks: readonly CertificationCheckEvidence[]
|
|
44
61
|
readonly capabilities: readonly CertificationCapabilityEvidence[]
|
|
45
62
|
readonly evidenceHash: string
|
|
@@ -54,6 +71,7 @@ export class AdapterCertificationError extends Error {
|
|
|
54
71
|
}
|
|
55
72
|
|
|
56
73
|
const TOKEN = /^[a-z0-9][a-z0-9._:-]{0,127}$/
|
|
74
|
+
const TARGET_NAME = /^@?[a-z0-9][a-z0-9._:-]*(?:\/[a-z0-9][a-z0-9._:-]*)?$/
|
|
57
75
|
const MAX_CHECKS = 128
|
|
58
76
|
|
|
59
77
|
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
@@ -69,6 +87,28 @@ function canonical(value: unknown): string {
|
|
|
69
87
|
return JSON.stringify(value)
|
|
70
88
|
}
|
|
71
89
|
|
|
90
|
+
const boundedTargetText = (value: string): boolean =>
|
|
91
|
+
value.length >= 1 &&
|
|
92
|
+
value.length <= 256 &&
|
|
93
|
+
[...value].every((character) => {
|
|
94
|
+
const code = character.codePointAt(0) ?? 0
|
|
95
|
+
return code > 0x1f && code !== 0x7f
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
function validateTarget(target: CertificationTarget | undefined): void {
|
|
99
|
+
if (target === undefined) return
|
|
100
|
+
if (target.adapter !== undefined && !TARGET_NAME.test(target.adapter))
|
|
101
|
+
throw new TypeError("certification target adapter must be a bounded package name")
|
|
102
|
+
for (const value of [target.runtime, target.witnessKind]) {
|
|
103
|
+
if (value !== undefined && !TOKEN.test(value))
|
|
104
|
+
throw new TypeError("certification target tokens must be bounded lowercase names")
|
|
105
|
+
}
|
|
106
|
+
for (const value of [target.artifact, target.source]) {
|
|
107
|
+
if (value !== undefined && !boundedTargetText(value))
|
|
108
|
+
throw new TypeError("certification target paths must be bounded and control-free")
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
72
112
|
async function sha256(value: string): Promise<string> {
|
|
73
113
|
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value))
|
|
74
114
|
return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("")
|
|
@@ -97,6 +137,7 @@ function validateProfile<Adapter>(profile: AdapterCertificationProfile<Adapter>)
|
|
|
97
137
|
`certification check ${check.id} names undeclared capability ${check.capability}`,
|
|
98
138
|
)
|
|
99
139
|
ids.add(check.id)
|
|
140
|
+
validateTarget(check.target)
|
|
100
141
|
}
|
|
101
142
|
for (const capability of capabilities) {
|
|
102
143
|
if (!profile.checks.some((check) => check.capability === capability))
|
|
@@ -115,27 +156,40 @@ export function defineCertificationProfile<Adapter>(
|
|
|
115
156
|
export async function certifyAdapter<Adapter>(options: {
|
|
116
157
|
readonly profile: AdapterCertificationProfile<Adapter>
|
|
117
158
|
readonly adapterId: string
|
|
159
|
+
/** Optional identity shared by all checks; a check target can add or override its fields. */
|
|
160
|
+
readonly target?: CertificationTarget
|
|
118
161
|
/** A fresh adapter per check prevents one failed check from contaminating the next. */
|
|
119
162
|
readonly createAdapter: () => Adapter | Promise<Adapter>
|
|
120
163
|
readonly cleanup?: (adapter: Adapter) => void | Promise<void>
|
|
121
164
|
}): Promise<AdapterCertificationReport> {
|
|
122
165
|
validateProfile(options.profile)
|
|
123
166
|
if (!TOKEN.test(options.adapterId)) throw new TypeError("adapter id must be a bounded token")
|
|
167
|
+
validateTarget(options.target)
|
|
124
168
|
const checks: CertificationCheckEvidence[] = []
|
|
125
169
|
for (const check of options.profile.checks) {
|
|
126
170
|
let adapter: Adapter | undefined
|
|
171
|
+
const target =
|
|
172
|
+
options.target === undefined && check.target === undefined
|
|
173
|
+
? undefined
|
|
174
|
+
: { ...options.target, ...check.target }
|
|
127
175
|
try {
|
|
128
176
|
adapter = await options.createAdapter()
|
|
129
177
|
await check.run(adapter, {
|
|
130
178
|
key: (suffix) =>
|
|
131
179
|
`nifra-cert:${options.profile.id}:${options.adapterId}:${check.id}:${suffix}`,
|
|
132
180
|
})
|
|
133
|
-
checks.push({
|
|
181
|
+
checks.push({
|
|
182
|
+
id: check.id,
|
|
183
|
+
capability: check.capability,
|
|
184
|
+
ok: true,
|
|
185
|
+
...(target === undefined ? {} : { target }),
|
|
186
|
+
})
|
|
134
187
|
} catch (error) {
|
|
135
188
|
checks.push({
|
|
136
189
|
id: check.id,
|
|
137
190
|
capability: check.capability,
|
|
138
191
|
ok: false,
|
|
192
|
+
...(target === undefined ? {} : { target }),
|
|
139
193
|
errorName:
|
|
140
194
|
error instanceof Error && TOKEN.test(error.name.toLowerCase())
|
|
141
195
|
? error.name
|
|
@@ -165,6 +219,7 @@ export async function certifyAdapter<Adapter>(options: {
|
|
|
165
219
|
ok: checks.every((check) => check.ok),
|
|
166
220
|
profile: { id: options.profile.id, version: options.profile.version },
|
|
167
221
|
adapterId: options.adapterId,
|
|
222
|
+
...(options.target === undefined ? {} : { target: options.target }),
|
|
168
223
|
checks,
|
|
169
224
|
capabilities,
|
|
170
225
|
}
|
|
@@ -256,6 +311,7 @@ export interface CertifiableStoredJob {
|
|
|
256
311
|
readonly payload: unknown
|
|
257
312
|
readonly attempt: number
|
|
258
313
|
readonly maxAttempts: number
|
|
314
|
+
readonly traceparent?: string
|
|
259
315
|
}
|
|
260
316
|
|
|
261
317
|
export interface CertifiableJobStore {
|
|
@@ -264,6 +320,7 @@ export interface CertifiableJobStore {
|
|
|
264
320
|
payload: unknown
|
|
265
321
|
runAt: number
|
|
266
322
|
maxAttempts: number
|
|
323
|
+
traceparent?: string
|
|
267
324
|
}): string | Promise<string>
|
|
268
325
|
lease(
|
|
269
326
|
now: number,
|
|
@@ -282,54 +339,90 @@ export interface CertifiableJobStore {
|
|
|
282
339
|
}>
|
|
283
340
|
}
|
|
284
341
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
342
|
+
/**
|
|
343
|
+
* The job-store profile. `traceparent: true` adds the optional `traceparent-roundtrip` capability: the
|
|
344
|
+
* trace context given to `enqueue` comes back on every lease, including after a retry. A store without
|
|
345
|
+
* it still runs jobs; traced runs just lose their link to the producer.
|
|
346
|
+
*/
|
|
347
|
+
export function jobStoreCertificationProfile(
|
|
348
|
+
options: { readonly traceparent?: boolean } = {},
|
|
349
|
+
): AdapterCertificationProfile<CertifiableJobStore> {
|
|
350
|
+
const capabilities = ["lease-complete", "retry-schedule", "dead-letter"]
|
|
351
|
+
const checks: CertificationCheck<CertifiableJobStore>[] = [
|
|
352
|
+
{
|
|
353
|
+
id: "lease-complete",
|
|
354
|
+
capability: "lease-complete",
|
|
355
|
+
async run(store) {
|
|
356
|
+
const id = await store.enqueue({
|
|
357
|
+
name: "cert",
|
|
358
|
+
payload: { n: 1 },
|
|
359
|
+
runAt: 10,
|
|
360
|
+
maxAttempts: 3,
|
|
361
|
+
})
|
|
362
|
+
if ((await store.lease(9, 1, 100)).length !== 0) throw new Error("JobEarlyLease")
|
|
363
|
+
const leased = await store.lease(10, 1, 100)
|
|
364
|
+
if (leased.length !== 1 || leased[0]?.id !== id || leased[0]?.attempt !== 0)
|
|
365
|
+
throw new Error("JobLease")
|
|
366
|
+
await store.complete(id)
|
|
367
|
+
if ((await store.counts()).pending !== 0) throw new Error("JobComplete")
|
|
308
368
|
},
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
369
|
+
},
|
|
370
|
+
{
|
|
371
|
+
id: "retry-schedule",
|
|
372
|
+
capability: "retry-schedule",
|
|
373
|
+
async run(store) {
|
|
374
|
+
const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 3 })
|
|
375
|
+
await store.lease(0, 1, 10)
|
|
376
|
+
await store.retry(id, 20)
|
|
377
|
+
if ((await store.lease(19, 1, 10)).length !== 0) throw new Error("JobRetryEarly")
|
|
378
|
+
const retried = await store.lease(20, 1, 10)
|
|
379
|
+
if (retried[0]?.attempt !== 1) throw new Error("JobRetryAttempt")
|
|
320
380
|
},
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
381
|
+
},
|
|
382
|
+
{
|
|
383
|
+
id: "dead-letter",
|
|
384
|
+
capability: "dead-letter",
|
|
385
|
+
async run(store) {
|
|
386
|
+
const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 1 })
|
|
387
|
+
await store.deadLetter(id, "bounded-code")
|
|
388
|
+
const counts = await store.counts()
|
|
389
|
+
if (counts.dead !== 1 || counts.pending !== 0) throw new Error("JobDeadLetter")
|
|
330
390
|
},
|
|
331
|
-
|
|
391
|
+
},
|
|
392
|
+
]
|
|
393
|
+
if (options.traceparent === true) {
|
|
394
|
+
capabilities.push("traceparent-roundtrip")
|
|
395
|
+
checks.push({
|
|
396
|
+
id: "traceparent-roundtrip",
|
|
397
|
+
capability: "traceparent-roundtrip",
|
|
398
|
+
async run(store) {
|
|
399
|
+
const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"
|
|
400
|
+
const id = await store.enqueue({
|
|
401
|
+
name: "cert",
|
|
402
|
+
payload: null,
|
|
403
|
+
runAt: 0,
|
|
404
|
+
maxAttempts: 3,
|
|
405
|
+
traceparent,
|
|
406
|
+
})
|
|
407
|
+
const first = await store.lease(0, 1, 10)
|
|
408
|
+
if (first[0]?.id !== id || first[0]?.traceparent !== traceparent)
|
|
409
|
+
throw new Error("JobTraceparentLease")
|
|
410
|
+
await store.retry(id, 20)
|
|
411
|
+
const retried = await store.lease(20, 1, 10)
|
|
412
|
+
if (retried[0]?.traceparent !== traceparent) throw new Error("JobTraceparentRetry")
|
|
413
|
+
const untraced = await store.enqueue({
|
|
414
|
+
name: "cert",
|
|
415
|
+
payload: null,
|
|
416
|
+
runAt: 0,
|
|
417
|
+
maxAttempts: 1,
|
|
418
|
+
})
|
|
419
|
+
const plain = (await store.lease(100, 2, 10)).find((job) => job.id === untraced)
|
|
420
|
+
if (plain === undefined || plain.traceparent !== undefined)
|
|
421
|
+
throw new Error("JobTraceparentAbsent")
|
|
422
|
+
},
|
|
423
|
+
})
|
|
332
424
|
}
|
|
425
|
+
return { id: "job-store", version: 1, capabilities, checks }
|
|
333
426
|
}
|
|
334
427
|
|
|
335
428
|
export interface CertifiableStorageObject {
|
package/src/contract-lab.ts
CHANGED
|
@@ -101,7 +101,7 @@ function canonical(value: unknown): string {
|
|
|
101
101
|
function witnessRequest(origin: string, witness: ContractLabWitness): Request {
|
|
102
102
|
return new Request(`${origin}${witness.request.path}`, {
|
|
103
103
|
method: witness.request.method,
|
|
104
|
-
headers: witness.request.headers,
|
|
104
|
+
...(witness.request.headers === undefined ? {} : { headers: witness.request.headers }),
|
|
105
105
|
...(witness.request.body === undefined ? {} : { body: witness.request.body }),
|
|
106
106
|
})
|
|
107
107
|
}
|