@nifrajs/testing 3.5.0 → 4.0.0

package/src/adversarial.ts

@@ -7,6 +7,11 @@
  * responses are validated off the hot path as part of the same laboratory run.
  */
 
+import {
+  compileRoutePattern,
+  type ParamConstraint,
+  type RoutePatternSegment,
+} from "@nifrajs/core/pattern"
 import {
   type JsonSchema,
   type ReflectedRoute,
@@ -26,6 +31,7 @@ type HeaderSource = ConstructorParameters<typeof Headers>[0]
 type InputTarget = "body" | "query"
 export type ContractTarget = "body" | "query" | "response"
 export type ContractCaseKind = "input-rejection" | "response-conformance"
+export type ContractAuthContext = "authenticated" | "unauthenticated"
 
 /** Anything that exposes reflected routes and a Web-standard in-process fetch handler. */
 export interface ContractTestApp {
@@ -45,6 +51,18 @@ export interface ContractWitness {
   readonly query?: unknown
   readonly body?: unknown
   readonly headers?: HeaderSource
+  /** Explicit auth state for the prepareRequest hook; the harness never invents credentials. */
+  readonly auth?: ContractAuthContext
+  /** Expected response for a guard/error witness. Omit for the normal success witness. */
+  readonly expected?: ContractExpectedResponse
+}
+
+export interface ContractExpectedResponse {
+  readonly status: number
+  readonly contentType?: "json" | "text"
+  readonly headers?: Readonly<Record<string, string>>
+  /** When present, the decoded body is compared byte-structure-wise to this value. */
+  readonly body?: unknown
 }
 
 /** Stable context passed to request/rejection hooks. It contains no request payloads or secrets. */
@@ -55,6 +73,7 @@ export interface ContractCaseContext {
   readonly runtime: string
   readonly kind: ContractCaseKind
   readonly target: ContractTarget
+  readonly auth?: ContractAuthContext
   readonly mutation?: string
 }
 
@@ -121,6 +140,7 @@ export type ContractCoverageGapCode =
   | "WITNESS_TOO_LARGE"
   | "UNSUPPORTED_BODY_METHOD"
   | "NO_REJECTED_MUTATION"
+  | "NO_STATUS_CONTRACT"
   | "NO_RUNTIME"
   | "INVALID_RUNTIME"
   | "CASE_NOT_FOUND"
@@ -257,6 +277,19 @@ const jsonClone = (value: unknown): unknown => {
   return JSON.parse(serialized) as unknown
 }
 
+const canonical = (value: unknown): string => {
+  if (value === undefined) return "undefined"
+  if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`
+  if (value !== null && typeof value === "object") {
+    const record = value as Readonly<Record<string, unknown>>
+    return `{${Object.keys(record)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`)
+      .join(",")}}`
+  }
+  return JSON.stringify(value)
+}
+
 const valueSize = (value: unknown): number => new TextEncoder().encode(JSON.stringify(value)).length
 
 const displayPath = (target: InputTarget, path: readonly (string | number)[]): string =>
@@ -268,7 +301,14 @@ const displayPath = (target: InputTarget, path: readonly (string | number)[]): s
 const mutationId = (routeKey: string, target: InputTarget, mutation: Mutation): string =>
   `${routeKey} :: ${displayPath(target, mutation.path)} :: ${mutation.reason}`
 
-const responseId = (routeKey: string): string => `${routeKey} :: response-conformance`
+const responseId = (routeKey: string, witness: ContractWitness): string => {
+  if (witness.auth === undefined && witness.expected === undefined) {
+    return `${routeKey} :: response-conformance`
+  }
+  const auth = witness.auth ?? "default"
+  const status = witness.expected?.status ?? "success"
+  return `${routeKey} :: ${auth} :: ${status} :: response-conformance`
+}
 
 const typeKind = (value: unknown): string => {
   if (value === null) return "null"
@@ -621,25 +661,48 @@ async function resolveInput(
   }
 }
 
-const materializePath = (
+/**
+ * A value `constraint` accepts: its first listed value, or one accepted character repeated to the
+ * shortest accepted length. Every character a constraint can name is legal in a path as written.
+ */
+function satisfying(constraint: ParamConstraint): string {
+  if (constraint.oneOf !== undefined) return constraint.oneOf[0]!
+  // Digits and letters first, so the usual classes give a value that reads like one.
+  let code = 48
+  while (constraint.mask[code] !== "1") code = code === 127 ? 33 : code + 1
+  return String.fromCharCode(code).repeat(constraint.min)
+}
+
+/** Params filled from the witness or a placeholder the route accepts, split by the router's compiler. */
+function materializePath(
   path: string,
   params: Readonly<Record<string, string>> | undefined,
-): string =>
-  path
-    .split("/")
-    .map((segment) => {
-      if (segment.startsWith(":")) {
-        const name = segment.slice(1)
-        return encodeURIComponent(params?.[name] ?? `${name || "param"}-contract`)
-      }
-      if (segment.startsWith("*")) {
-        const name = segment.slice(1)
-        const value = params?.[name] ?? `${name || "path"}/contract`
-        return value.split("/").filter(Boolean).map(encodeURIComponent).join("/")
-      }
-      return segment
-    })
-    .join("/")
+): string {
+  let segments: readonly RoutePatternSegment[]
+  try {
+    segments = compileRoutePattern(path).segments
+  } catch {
+    return path
+  }
+  const fill = (name: string, constraint?: ParamConstraint): string => {
+    const given = params?.[name]
+    if (given !== undefined) return encodeURIComponent(given)
+    return constraint === undefined ? `${name}-contract` : satisfying(constraint)
+  }
+  const filled = segments.map((segment) => {
+    if (segment.kind === "static") return segment.value
+    if (segment.kind === "param") return fill(segment.name)
+    if (segment.kind === "mixed") {
+      return segment.parts
+        .map((part) => (part.t === "lit" ? part.v : fill(part.name, part.c)))
+        .join("")
+    }
+    const value =
+      params?.[segment.name] ?? `${segment.name === "*" ? "path" : segment.name}/contract`
+    return value.split("/").filter(Boolean).map(encodeURIComponent).join("/")
+  })
+  return `/${filled.join("/")}`
+}
 
 function requestFor(
   laboratory: RouteLaboratory,
@@ -721,6 +784,7 @@ async function executeRejection(
     runtime: runtime.name,
     kind: "input-rejection",
     target,
+    ...(laboratory.witness.auth === undefined ? {} : { auth: laboratory.witness.auth }),
     mutation: `${displayPath(target, mutation.path)} ${mutation.reason}`,
   }
   try {
@@ -868,7 +932,12 @@ async function buildLaboratories(
       if (resolved.ok) query = resolved.input
       else gaps.push(resolved.gap)
     }
-    if (options.validateResponses !== false && route.schema?.response !== undefined)
+    const expectedStatus = witness.expected?.status
+    const expectedResponseSchema =
+      expectedStatus !== undefined && (expectedStatus < 200 || expectedStatus >= 300)
+        ? route.schema?.errors?.[expectedStatus]
+        : route.schema?.response
+    if (options.validateResponses !== false && expectedResponseSchema !== undefined)
       targetCount += 1
 
     const allInputsResolved =
@@ -1027,10 +1096,25 @@ export async function runAdversarialContract(
       }
     }
 
-    const responseSchema = laboratory.route.schema?.response
-    if (options.validateResponses === false || responseSchema === undefined) continue
-    const id = responseId(laboratory.routeKey)
+    const expected = laboratory.witness.expected
+    const responseSchema =
+      expected !== undefined && (expected.status < 200 || expected.status >= 300)
+        ? laboratory.route.schema?.errors?.[expected.status]
+        : laboratory.route.schema?.response
+    if (options.validateResponses === false) continue
+    const id = responseId(laboratory.routeKey, laboratory.witness)
     if (only !== undefined && !only.has(id)) continue
+    if (responseSchema === undefined) {
+      if (expected !== undefined && (expected.status < 200 || expected.status >= 300)) {
+        gaps.push({
+          route: laboratory.routeKey,
+          target: "response",
+          code: "NO_STATUS_CONTRACT",
+          message: `expected status ${expected.status} has no declared error response contract`,
+        })
+      }
+      continue
+    }
     if (responseSchema.standard === undefined) {
       gaps.push({
         route: laboratory.routeKey,
@@ -1053,6 +1137,7 @@ export async function runAdversarialContract(
         runtime: runtime.name,
         kind: "response-conformance",
         target: "response",
+        ...(laboratory.witness.auth === undefined ? {} : { auth: laboratory.witness.auth }),
       }
       const base = {
         id,
@@ -1069,13 +1154,15 @@ export async function runAdversarialContract(
         )
         const response = await runtime.fetch(request)
         const status = response.status
-        if (!response.ok) {
+        if (expected === undefined ? !response.ok : response.status !== expected.status) {
           await discardResponse(response)
           results.push(
             resultForFailure(
               { ...base, status },
               seed,
-              `valid witness returned non-success status ${status}`,
+              expected === undefined
+                ? `valid witness returned non-success status ${status}`
+                : `expected status ${expected.status}, received ${status}`,
             ),
           )
           continue
@@ -1083,7 +1170,12 @@ export async function runAdversarialContract(
         const text = await response.text()
         let value: unknown
         try {
-          value = text.length === 0 ? undefined : (JSON.parse(text) as unknown)
+          value =
+            expected?.contentType === "text"
+              ? text
+              : text.length === 0
+                ? undefined
+                : (JSON.parse(text) as unknown)
         } catch {
           results.push(
             resultForFailure(
@@ -1094,6 +1186,26 @@ export async function runAdversarialContract(
           )
           continue
         }
+        if (expected !== undefined) {
+          const expectedContentType =
+            expected.contentType === "text" ? "text/plain" : "application/json"
+          const actualContentType = response.headers.get("content-type")?.split(";", 1)[0] ?? ""
+          const headersMatch = Object.entries(expected.headers ?? {}).every(
+            ([name, headerValue]) => response.headers.get(name) === headerValue,
+          )
+          const bodyMatch =
+            !Object.hasOwn(expected, "body") || canonical(value) === canonical(expected.body)
+          if (actualContentType !== expectedContentType || !headersMatch || !bodyMatch) {
+            results.push(
+              resultForFailure(
+                { ...base, status },
+                seed,
+                `expected response envelope ${expectedContentType} ${canonical(expected.body)}, received ${actualContentType || "(missing content type)"} ${canonical(value)}`,
+              ),
+            )
+            continue
+          }
+        }
         const validation = await responseSchema.standard["~standard"].validate(value)
         if (validation.issues !== undefined) {
           const first = validation.issues[0]
@@ -1153,12 +1265,16 @@ export async function runAdversarialContract(
   if (
     options.validateResponses !== false &&
     routes.length > 0 &&
-    !laboratories.some((laboratory) => laboratory.route.schema?.response !== undefined)
+    !laboratories.some(
+      (laboratory) =>
+        laboratory.route.schema?.response !== undefined ||
+        Object.keys(laboratory.route.schema?.errors ?? {}).length > 0,
+    )
   ) {
     advisories.push(
-      "validateResponses is on, but no selected route declares a `response` schema, so response " +
+      "validateResponses is on, but no selected route declares a `response` or status-specific `errors` schema, so response " +
         "conformance checked 0 targets. Response types inferred from a handler's return are a client-side " +
-        "contract only - declare `response:` on the routes you want the laboratory to verify.",
+        "contract only - declare `response:` or `errors:` on the routes you want the laboratory to verify.",
     )
   }
   return {

package/src/agent-eval.ts

@@ -17,6 +17,9 @@ import { type FailureDirective, FailureInjectedError, runFailureScenario } from
 import { type FaultProfile, runFaultProfile } from "./fault-profile.ts"
 import { type EffectLedger, type IdempotencyProof, proveIdempotency } from "./idempotency.ts"
 
+/** Code-unit order, the same in every locale: a case digest must not depend on the machine scoring it. */
+const byCodeUnit = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0)
+
 const TOKEN = /^[a-z0-9][a-z0-9._:-]{0,127}$/
 const MAX_CASES = 512
 const MAX_RUBRICS = 32
@@ -509,9 +512,7 @@ export function defineAgentEvalSuite(spec: AgentEvalSuiteSpec): AgentEvalSuite {
             throw new TypeError(`agent eval: case '${caseId}' scored rubric '${rubric.id}' twice`)
           perRubric.set(rubric.id, parseRubricVerdict(rubric, raw))
         }
-        const verdicts = [...perRubric.values()].sort((a, b) =>
-          a.rubricId.localeCompare(b.rubricId),
-        )
+        const verdicts = [...perRubric.values()].sort((a, b) => byCodeUnit(a.rubricId, b.rubricId))
         cases.push({
           caseId,
           verdicts: Object.freeze(verdicts),

package/src/certification.ts

@@ -311,6 +311,7 @@ export interface CertifiableStoredJob {
   readonly payload: unknown
   readonly attempt: number
   readonly maxAttempts: number
+  readonly traceparent?: string
 }
 
 export interface CertifiableJobStore {
@@ -319,6 +320,7 @@ export interface CertifiableJobStore {
     payload: unknown
     runAt: number
     maxAttempts: number
+    traceparent?: string
   }): string | Promise<string>
   lease(
     now: number,
@@ -337,54 +339,90 @@ export interface CertifiableJobStore {
       }>
 }
 
-export function jobStoreCertificationProfile(): AdapterCertificationProfile<CertifiableJobStore> {
-  return {
-    id: "job-store",
-    version: 1,
-    capabilities: ["lease-complete", "retry-schedule", "dead-letter"],
-    checks: [
-      {
-        id: "lease-complete",
-        capability: "lease-complete",
-        async run(store) {
-          const id = await store.enqueue({
-            name: "cert",
-            payload: { n: 1 },
-            runAt: 10,
-            maxAttempts: 3,
-          })
-          if ((await store.lease(9, 1, 100)).length !== 0) throw new Error("JobEarlyLease")
-          const leased = await store.lease(10, 1, 100)
-          if (leased.length !== 1 || leased[0]?.id !== id || leased[0]?.attempt !== 0)
-            throw new Error("JobLease")
-          await store.complete(id)
-          if ((await store.counts()).pending !== 0) throw new Error("JobComplete")
-        },
+/**
+ * The job-store profile. `traceparent: true` adds the optional `traceparent-roundtrip` capability: the
+ * trace context given to `enqueue` comes back on every lease, including after a retry. A store without
+ * it still runs jobs; traced runs just lose their link to the producer.
+ */
+export function jobStoreCertificationProfile(
+  options: { readonly traceparent?: boolean } = {},
+): AdapterCertificationProfile<CertifiableJobStore> {
+  const capabilities = ["lease-complete", "retry-schedule", "dead-letter"]
+  const checks: CertificationCheck<CertifiableJobStore>[] = [
+    {
+      id: "lease-complete",
+      capability: "lease-complete",
+      async run(store) {
+        const id = await store.enqueue({
+          name: "cert",
+          payload: { n: 1 },
+          runAt: 10,
+          maxAttempts: 3,
+        })
+        if ((await store.lease(9, 1, 100)).length !== 0) throw new Error("JobEarlyLease")
+        const leased = await store.lease(10, 1, 100)
+        if (leased.length !== 1 || leased[0]?.id !== id || leased[0]?.attempt !== 0)
+          throw new Error("JobLease")
+        await store.complete(id)
+        if ((await store.counts()).pending !== 0) throw new Error("JobComplete")
       },
-      {
-        id: "retry-schedule",
-        capability: "retry-schedule",
-        async run(store) {
-          const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 3 })
-          await store.lease(0, 1, 10)
-          await store.retry(id, 20)
-          if ((await store.lease(19, 1, 10)).length !== 0) throw new Error("JobRetryEarly")
-          const retried = await store.lease(20, 1, 10)
-          if (retried[0]?.attempt !== 1) throw new Error("JobRetryAttempt")
-        },
+    },
+    {
+      id: "retry-schedule",
+      capability: "retry-schedule",
+      async run(store) {
+        const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 3 })
+        await store.lease(0, 1, 10)
+        await store.retry(id, 20)
+        if ((await store.lease(19, 1, 10)).length !== 0) throw new Error("JobRetryEarly")
+        const retried = await store.lease(20, 1, 10)
+        if (retried[0]?.attempt !== 1) throw new Error("JobRetryAttempt")
       },
-      {
-        id: "dead-letter",
-        capability: "dead-letter",
-        async run(store) {
-          const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 1 })
-          await store.deadLetter(id, "bounded-code")
-          const counts = await store.counts()
-          if (counts.dead !== 1 || counts.pending !== 0) throw new Error("JobDeadLetter")
-        },
+    },
+    {
+      id: "dead-letter",
+      capability: "dead-letter",
+      async run(store) {
+        const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 1 })
+        await store.deadLetter(id, "bounded-code")
+        const counts = await store.counts()
+        if (counts.dead !== 1 || counts.pending !== 0) throw new Error("JobDeadLetter")
       },
-    ],
+    },
+  ]
+  if (options.traceparent === true) {
+    capabilities.push("traceparent-roundtrip")
+    checks.push({
+      id: "traceparent-roundtrip",
+      capability: "traceparent-roundtrip",
+      async run(store) {
+        const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"
+        const id = await store.enqueue({
+          name: "cert",
+          payload: null,
+          runAt: 0,
+          maxAttempts: 3,
+          traceparent,
+        })
+        const first = await store.lease(0, 1, 10)
+        if (first[0]?.id !== id || first[0]?.traceparent !== traceparent)
+          throw new Error("JobTraceparentLease")
+        await store.retry(id, 20)
+        const retried = await store.lease(20, 1, 10)
+        if (retried[0]?.traceparent !== traceparent) throw new Error("JobTraceparentRetry")
+        const untraced = await store.enqueue({
+          name: "cert",
+          payload: null,
+          runAt: 0,
+          maxAttempts: 1,
+        })
+        const plain = (await store.lease(100, 2, 10)).find((job) => job.id === untraced)
+        if (plain === undefined || plain.traceparent !== undefined)
+          throw new Error("JobTraceparentAbsent")
+      },
+    })
   }
+  return { id: "job-store", version: 1, capabilities, checks }
 }
 
 export interface CertifiableStorageObject {

package/src/contract-lab.ts

@@ -101,7 +101,7 @@ function canonical(value: unknown): string {
 function witnessRequest(origin: string, witness: ContractLabWitness): Request {
   return new Request(`${origin}${witness.request.path}`, {
     method: witness.request.method,
-    headers: witness.request.headers,
+    ...(witness.request.headers === undefined ? {} : { headers: witness.request.headers }),
     ...(witness.request.body === undefined ? {} : { body: witness.request.body }),
   })
 }