@nifrajs/testing 4.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/agent-eval.ts CHANGED
@@ -106,7 +106,10 @@ export type ComparisonCode =
106
106
  | "improved"
107
107
  | "tolerated"
108
108
  | "regressed"
109
+ /** The baseline has a verdict the current report no longer produces. */
109
110
  | "missing"
111
+ /** The current report has a verdict the baseline lacks. */
112
+ | "added"
110
113
  | "incomparable"
111
114
 
112
115
  /** Explicit tolerance for a score regression. `rubricId` omitted applies to every rubric. */
@@ -120,7 +123,10 @@ export interface ScoreTolerance {
120
123
 
121
124
  export interface BaselineOptions {
122
125
  readonly tolerances?: readonly ScoreTolerance[]
123
- /** Comparison codes that fail the assertion. Default `["regressed"]`. */
126
+ /**
127
+ * Comparison codes that fail the assertion. Default `["regressed", "missing", "incomparable"]`:
128
+ * a verdict that disappeared or whose rubric changed shape cannot show it did not regress.
129
+ */
124
130
  readonly failOn?: readonly ComparisonCode[]
125
131
  }
126
132
 
@@ -207,13 +213,26 @@ function matrixSchedule(caseId: string): readonly FailureDirective[] {
207
213
  return [{ point: `agent.${caseId}`, kind: "crash" }]
208
214
  }
209
215
 
216
+ export interface AgentFailureMatrixOptions {
217
+ readonly seed?: number
218
+ /**
219
+ * Drive the agent under test across one boundary, calling `checkpoint()` where that boundary's
220
+ * fault lands. The case passes only when the injected crash surfaces from `execute`. Without it
221
+ * each case runs a bare fixture, which proves the schedule and replay identity, not an agent.
222
+ */
223
+ readonly execute?: (
224
+ boundary: { readonly id: string; readonly kind: AgentFailureMatrixKind },
225
+ checkpoint: () => void,
226
+ ) => unknown
227
+ }
228
+
210
229
  /**
211
- * Run the same token-only crash fixture at each agent boundary. The implementation intentionally
212
- * delegates scheduling and replay identity to the existing failure laboratory; this is a matrix
213
- * consumer, not a second fault engine.
230
+ * Inject a crash at each agent boundary. The implementation intentionally delegates scheduling and
231
+ * replay identity to the existing failure laboratory; this is a matrix consumer, not a second fault
232
+ * engine.
214
233
  */
215
234
  export async function runAgentFailureMatrix(
216
- options: { readonly seed?: number } = {},
235
+ options: AgentFailureMatrixOptions = {},
217
236
  ): Promise<AgentFailureMatrixReport> {
218
237
  const seed = Number.isFinite(options.seed) ? Math.trunc(options.seed as number) >>> 0 : 0x4e494652
219
238
  const cases: AgentFailureMatrixCase[] = []
@@ -223,7 +242,9 @@ export async function runAgentFailureMatrix(
223
242
  {
224
243
  name: `agent-${caseId}`,
225
244
  execute(lab) {
226
- lab.checkpoint(`agent.${caseId}`)
245
+ const checkpoint = (): void => lab.checkpoint(`agent.${caseId}`)
246
+ if (options.execute !== undefined) return options.execute(entry, checkpoint)
247
+ checkpoint()
227
248
  return true
228
249
  },
229
250
  verify: ({ error }) => error instanceof FailureInjectedError,
@@ -590,7 +611,7 @@ export async function compareAgentEvalBaseline(
590
611
  }),
591
612
  )
592
613
  }
593
- const failOn = new Set(options.failOn ?? ["regressed"])
614
+ const failOn = new Set(options.failOn ?? ["regressed", "missing", "incomparable"])
594
615
  const regressions = comparisons
595
616
  .filter((entry) => failOn.has(entry.code))
596
617
  .map((entry) => entry.regressionId)
@@ -629,7 +650,8 @@ function classify(
629
650
  rubricId: string,
630
651
  options: BaselineOptions,
631
652
  ): ComparisonCode {
632
- if (base === undefined || cur === undefined) return "missing"
653
+ if (base === undefined) return "added"
654
+ if (cur === undefined) return "missing"
633
655
  if (baseShape === undefined || curShape === undefined || baseShape.digest !== curShape.digest)
634
656
  return "incomparable"
635
657
  const baseRank = curShape.outcomes.indexOf(base.outcome)
@@ -252,57 +252,99 @@ export interface CertifiableCacheStore {
252
252
  delete(key: string): void | Promise<void>
253
253
  invalidateTag(tag: string): void | Promise<void>
254
254
  clear(): void | Promise<void>
255
+ fence?(key: string, tags: readonly string[]): unknown | Promise<unknown>
256
+ setIfFence?(
257
+ key: string,
258
+ entry: CertifiableCacheEntry,
259
+ tags: readonly string[],
260
+ token: unknown,
261
+ ): boolean | Promise<boolean>
255
262
  }
256
263
 
257
- export function cacheStoreCertificationProfile(): AdapterCertificationProfile<CertifiableCacheStore> {
264
+ /**
265
+ * The cache-store profile. `fencing: true` adds the optional `fenced-write` capability: `setIfFence`
266
+ * refuses a write after an invalidation of its tag, a delete or set of its key, or a clear since its
267
+ * `fence`, and accepts one after an unrelated change. A store several caches share needs it, or a load
268
+ * running in one cache can store a value another cache just invalidated.
269
+ */
270
+ export function cacheStoreCertificationProfile(
271
+ options: { readonly fencing?: boolean } = {},
272
+ ): AdapterCertificationProfile<CertifiableCacheStore> {
273
+ // Fresh on any clock until 2100: a store that honors expiry on the wall clock must still keep it.
258
274
  const entry = (value: unknown): CertifiableCacheEntry => ({
259
275
  value,
260
- staleAt: 5_000_000,
261
- expiresAt: 6_000_000,
276
+ staleAt: 4_102_444_800_000,
277
+ expiresAt: 4_102_448_400_000,
262
278
  })
263
- return {
264
- id: "cache-store",
265
- version: 1,
266
- capabilities: ["roundtrip", "tag-invalidation", "clear"],
267
- checks: [
268
- {
269
- id: "set-get-delete",
270
- capability: "roundtrip",
271
- async run(store, context) {
272
- const key = context.key("roundtrip")
273
- await store.set(key, entry({ answer: 42 }), [])
274
- const found = await store.get(key)
275
- if (JSON.stringify(found?.value) !== '{"answer":42}') throw new Error("CacheRoundtrip")
276
- await store.delete(key)
277
- if ((await store.get(key)) !== undefined) throw new Error("CacheDelete")
278
- },
279
+ const capabilities = ["roundtrip", "tag-invalidation", "clear"]
280
+ const checks: CertificationCheck<CertifiableCacheStore>[] = [
281
+ {
282
+ id: "set-get-delete",
283
+ capability: "roundtrip",
284
+ async run(store, context) {
285
+ const key = context.key("roundtrip")
286
+ await store.set(key, entry({ answer: 42 }), [])
287
+ const found = await store.get(key)
288
+ if (JSON.stringify(found?.value) !== '{"answer":42}') throw new Error("CacheRoundtrip")
289
+ await store.delete(key)
290
+ if ((await store.get(key)) !== undefined) throw new Error("CacheDelete")
279
291
  },
280
- {
281
- id: "invalidate-tag",
282
- capability: "tag-invalidation",
283
- async run(store, context) {
284
- const tagged = context.key("tagged")
285
- const other = context.key("other")
286
- const tag = context.key("tag")
287
- await store.set(tagged, entry(1), [tag])
288
- await store.set(other, entry(2), [context.key("other-tag")])
289
- await store.invalidateTag(tag)
290
- if ((await store.get(tagged)) !== undefined || (await store.get(other))?.value !== 2)
291
- throw new Error("CacheTagInvalidation")
292
- },
292
+ },
293
+ {
294
+ id: "invalidate-tag",
295
+ capability: "tag-invalidation",
296
+ async run(store, context) {
297
+ const tagged = context.key("tagged")
298
+ const other = context.key("other")
299
+ const tag = context.key("tag")
300
+ await store.set(tagged, entry(1), [tag])
301
+ await store.set(other, entry(2), [context.key("other-tag")])
302
+ await store.invalidateTag(tag)
303
+ if ((await store.get(tagged)) !== undefined || (await store.get(other))?.value !== 2)
304
+ throw new Error("CacheTagInvalidation")
293
305
  },
294
- {
295
- id: "clear",
296
- capability: "clear",
297
- async run(store, context) {
298
- const key = context.key("clear")
299
- await store.set(key, entry(1), [])
300
- await store.clear()
301
- if ((await store.get(key)) !== undefined) throw new Error("CacheClear")
302
- },
306
+ },
307
+ {
308
+ id: "clear",
309
+ capability: "clear",
310
+ async run(store, context) {
311
+ const key = context.key("clear")
312
+ await store.set(key, entry(1), [])
313
+ await store.clear()
314
+ if ((await store.get(key)) !== undefined) throw new Error("CacheClear")
303
315
  },
304
- ],
316
+ },
317
+ ]
318
+ if (options.fencing === true) {
319
+ capabilities.push("fenced-write")
320
+ checks.push({
321
+ id: "fenced-write",
322
+ capability: "fenced-write",
323
+ async run(store, context) {
324
+ if (store.fence === undefined || store.setIfFence === undefined)
325
+ throw new Error("CacheFenceMissing")
326
+ const key = context.key("fenced")
327
+ const tag = context.key("fenced-tag")
328
+ const changes: readonly [() => void | Promise<void>, boolean][] = [
329
+ [() => store.set(context.key("unrelated"), entry(0), []), true],
330
+ [() => store.invalidateTag(tag), false],
331
+ [() => store.delete(key), false],
332
+ [() => store.set(key, entry("newer"), []), false],
333
+ [() => store.clear(), false],
334
+ ]
335
+ for (const [change, writes] of changes) {
336
+ await store.delete(key)
337
+ const token = await store.fence(key, [tag])
338
+ await change()
339
+ if ((await store.setIfFence(key, entry("loaded"), [tag], token)) !== writes)
340
+ throw new Error("CacheFencedWrite")
341
+ if (((await store.get(key))?.value === "loaded") !== writes)
342
+ throw new Error("CacheFencedWriteStored")
343
+ }
344
+ },
345
+ })
305
346
  }
347
+ return { id: "cache-store", version: 2, capabilities, checks }
306
348
  }
307
349
 
308
350
  export interface CertifiableStoredJob {
@@ -367,6 +409,19 @@ export function jobStoreCertificationProfile(
367
409
  if ((await store.counts()).pending !== 0) throw new Error("JobComplete")
368
410
  },
369
411
  },
412
+ {
413
+ // A lease hides the job from every other worker until it runs out, then hands it back.
414
+ id: "lease-exclusive",
415
+ capability: "lease-complete",
416
+ async run(store) {
417
+ const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 3 })
418
+ const first = await store.lease(0, 1, 100)
419
+ if (first.length !== 1 || first[0]?.id !== id) throw new Error("JobLease")
420
+ if ((await store.lease(50, 1, 100)).length !== 0) throw new Error("JobLeaseShared")
421
+ const expired = await store.lease(101, 1, 100)
422
+ if (expired.length !== 1 || expired[0]?.id !== id) throw new Error("JobLeaseExpiry")
423
+ },
424
+ },
370
425
  {
371
426
  id: "retry-schedule",
372
427
  capability: "retry-schedule",
@@ -422,7 +477,7 @@ export function jobStoreCertificationProfile(
422
477
  },
423
478
  })
424
479
  }
425
- return { id: "job-store", version: 1, capabilities, checks }
480
+ return { id: "job-store", version: 2, capabilities, checks }
426
481
  }
427
482
 
428
483
  export interface CertifiableStorageObject {
@@ -463,10 +518,24 @@ export interface CertifiableStorageAdapter {
463
518
  move?(sourceKey: string, destinationKey: string): Promise<void>
464
519
  }
465
520
 
521
+ /** Keys the storage key contract refuses: a `..` segment anywhere, an absolute path, a backslash, a
522
+ * NUL, and an empty, `.` or trailing segment. Refusing only `../escape` leaves the rest open. */
523
+ const UNSAFE_STORAGE_KEYS = [
524
+ "../escape",
525
+ "cert/../../escape",
526
+ "/cert/absolute",
527
+ "cert\\..\\escape",
528
+ "cert/nul\0",
529
+ "",
530
+ "cert//empty-segment",
531
+ "cert/./dot-segment",
532
+ "cert/trailing/",
533
+ ]
534
+
466
535
  export function storageAdapterCertificationProfile(
467
536
  options: { readonly paging?: boolean; readonly presign?: boolean; readonly move?: boolean } = {},
468
537
  ): AdapterCertificationProfile<CertifiableStorageAdapter> {
469
- const capabilities = ["object-roundtrip", "overwrite", "list-delete"]
538
+ const capabilities = ["object-roundtrip", "overwrite", "list-delete", "key-safety"]
470
539
  const checks: CertificationCheck<CertifiableStorageAdapter>[] = [
471
540
  {
472
541
  id: "object-roundtrip",
@@ -513,6 +582,29 @@ export function storageAdapterCertificationProfile(
513
582
  throw new Error("StorageDelete")
514
583
  },
515
584
  },
585
+ {
586
+ id: "unsafe-keys",
587
+ capability: "key-safety",
588
+ async run(store) {
589
+ const refuses = async (call: () => Promise<unknown>): Promise<boolean> => {
590
+ try {
591
+ await call()
592
+ return false
593
+ } catch {
594
+ return true
595
+ }
596
+ }
597
+ for (const key of UNSAFE_STORAGE_KEYS) {
598
+ const refused = await Promise.all([
599
+ refuses(() => store.put(key, "x")),
600
+ refuses(() => store.get(key)),
601
+ refuses(() => store.exists(key)),
602
+ refuses(() => store.delete(key)),
603
+ ])
604
+ if (refused.includes(false)) throw new Error("StorageUnsafeKey")
605
+ }
606
+ },
607
+ },
516
608
  ]
517
609
  if (options.paging === true) {
518
610
  capabilities.push("cursor-paging")
@@ -576,7 +668,7 @@ export function storageAdapterCertificationProfile(
576
668
  }
577
669
  return {
578
670
  id: "storage-adapter",
579
- version: 1,
671
+ version: 2,
580
672
  capabilities,
581
673
  checks,
582
674
  }
package/src/cookies.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * A tiny cookie jar for in-process tests - parses `Set-Cookie` off responses and emits a `Cookie`
3
3
  * request header, so a login → authenticated-request flow works without threading headers by hand. It
4
- * honours removal (`Max-Age=0` / a past `Expires`) so logout clears the cookie; other attributes
4
+ * honours removal (`Max-Age` of zero or less, or a past `Expires`) so logout clears the cookie; other attributes
5
5
  * (Domain/Path/Secure/SameSite) are ignored - everything is same-origin in-process.
6
6
  */
7
7
 
@@ -27,7 +27,9 @@ function setCookies(headers: Headers): string[] {
27
27
  }
28
28
 
29
29
  function isRemoval(attributes: string): boolean {
30
- if (/(^|;)\s*max-age=0(\s|;|$)/.test(attributes)) return true
30
+ // RFC 6265 5.2.2: any Max-Age of zero or less removes the cookie, and Max-Age outranks Expires.
31
+ const maxAge = /(?:^|;)\s*max-age=(-?\d+)\s*(?:;|$)/.exec(attributes)
32
+ if (maxAge !== null) return Number(maxAge[1]) <= 0
31
33
  const expires = /expires=([^;]+)/.exec(attributes)
32
34
  if (expires === null) return false
33
35
  const at = Date.parse(expires[1] as string)
@@ -215,19 +215,29 @@ export async function runDataAdapterConformance<Adapter, Row, Snapshot, Cursor =
215
215
  }
216
216
  })
217
217
 
218
+ const keysIn = async (scope: DataConformanceScope): Promise<ReadonlySet<string>> =>
219
+ new Set((await target.read(primary, scope)).map(target.rowKey))
220
+ const sameKeys = (a: ReadonlySet<string>, b: ReadonlySet<string>): boolean =>
221
+ a.size === b.size && [...a].every((key) => b.has(key))
222
+
218
223
  await run("cancellation", async () => {
219
224
  const controller = new AbortController()
220
225
  controller.abort()
226
+ let refused = false
221
227
  try {
222
228
  await target.write(primary, scopes.left, rows.rollback, controller.signal)
223
229
  } catch (error) {
224
- if (isAbortLike(error) || controller.signal.aborted) return
225
- throw error
230
+ // Any other failure (a refused connection, a rejected row) proves nothing about the signal.
231
+ if (!isAbortLike(error) && error !== controller.signal.reason) throw error
232
+ refused = true
226
233
  }
227
- throw new Error("cancelled write resolved successfully")
234
+ if (!refused) throw new Error("cancelled write resolved successfully")
235
+ if ((await keysIn(scopes.left)).has(target.rowKey(rows.rollback)))
236
+ throw new Error("cancelled write reached durable state")
228
237
  })
229
238
 
230
239
  await run("pagination", async () => {
240
+ const expected = await keysIn(scopes.left)
231
241
  const seen: string[] = []
232
242
  const cursors = new Set<string>()
233
243
  let cursor: Cursor | undefined
@@ -242,16 +252,22 @@ export async function runDataAdapterConformance<Adapter, Row, Snapshot, Cursor =
242
252
  if (pageNumber === 99) throw new Error("pagination did not terminate")
243
253
  }
244
254
  if (new Set(seen).size !== seen.length) throw new Error("pagination returned a duplicate row")
255
+ if (!sameKeys(new Set(seen), expected))
256
+ throw new Error("pagination did not return exactly the scope's rows")
245
257
  })
246
258
 
247
259
  await run("backup-restore", async () => {
260
+ const left = await keysIn(scopes.left)
261
+ const right = await keysIn(scopes.right)
248
262
  const snapshot = await target.snapshot(primary)
249
263
  await target.seed(primary, scopes.left, rows.rollback)
250
264
  await target.restore(primary, snapshot)
251
- const restored = new Set((await target.read(primary, scopes.left)).map(target.rowKey))
252
- if (restored.has(target.rowKey(rows.rollback))) {
253
- throw new Error("restore did not remove post-backup data")
254
- }
265
+ // Restore returns every scope to the snapshot: no later row, and no row the snapshot held lost.
266
+ if (
267
+ !sameKeys(await keysIn(scopes.left), left) ||
268
+ !sameKeys(await keysIn(scopes.right), right)
269
+ )
270
+ throw new Error("restore did not return the snapshot's rows")
255
271
  })
256
272
 
257
273
  await run("multi-instance", async () => {
package/src/incident.ts CHANGED
@@ -18,16 +18,23 @@ export interface CapturedRequestInput {
18
18
  readonly method?: string
19
19
  readonly path: string
20
20
  readonly query?: Record<string, string>
21
+ readonly queryPairs?: readonly (readonly [string, string])[]
21
22
  readonly headers?: Record<string, string>
22
23
  readonly body?: unknown
24
+ readonly bodyEncoding?: "text"
23
25
  }
24
26
 
25
27
  export interface CapturedRequest {
26
28
  readonly method: string
27
29
  readonly path: string
30
+ /** The query by name; a repeated name keeps its last value. */
28
31
  readonly query?: Record<string, string>
32
+ /** Every query pair in order, present when `query` cannot rebuild it. Replay prefers it. */
33
+ readonly queryPairs?: readonly (readonly [string, string])[]
29
34
  readonly headers?: Record<string, string>
30
35
  readonly body?: unknown
36
+ /** `"text"`: `body` is the request text as sent, and replay sends it verbatim. Absent: JSON. */
37
+ readonly bodyEncoding?: "text"
31
38
  }
32
39
 
33
40
  export interface IncidentCapsule {
@@ -95,6 +102,29 @@ async function readBody(source: Request | Response): Promise<unknown> {
95
102
  return text
96
103
  }
97
104
 
105
+ /** The request body as captured: parsed JSON when the request said so, its text otherwise. */
106
+ async function readRequestBody(
107
+ request: Request,
108
+ ): Promise<{ readonly body: unknown; readonly text: boolean } | undefined> {
109
+ const text = await request.clone().text()
110
+ if (text === "") return undefined
111
+ if ((request.headers.get("content-type") ?? "").includes("application/json")) {
112
+ try {
113
+ return { body: JSON.parse(text), text: false }
114
+ } catch {}
115
+ }
116
+ return { body: text, text: true }
117
+ }
118
+
119
+ function samePairs(
120
+ a: readonly (readonly [string, string])[],
121
+ b: readonly (readonly [string, string])[],
122
+ ): boolean {
123
+ return (
124
+ a.length === b.length && a.every(([key, value], i) => b[i]?.[0] === key && b[i]?.[1] === value)
125
+ )
126
+ }
127
+
98
128
  function pickHeaders(
99
129
  headers: Headers,
100
130
  allow: readonly string[],
@@ -118,25 +148,35 @@ export async function captureIncident(
118
148
  method: string
119
149
  path: string
120
150
  query?: Record<string, string>
151
+ queryPairs?: readonly (readonly [string, string])[]
121
152
  headers?: Record<string, string>
122
153
  body?: unknown
154
+ bodyEncoding?: "text"
123
155
  } = { method: "GET", path: "/" }
124
156
  if (request instanceof Request) {
125
157
  const url = new URL(request.url)
126
- const query = Object.fromEntries(url.searchParams.entries())
158
+ const pairs = [...url.searchParams.entries()]
159
+ const query = Object.fromEntries(pairs)
127
160
  const headers = pickHeaders(request.headers, allow)
128
- const body = await readBody(request)
161
+ const body = await readRequestBody(request)
129
162
  req.method = request.method
130
163
  req.path = url.pathname
131
- if (Object.keys(query).length > 0) req.query = query
164
+ if (pairs.length > 0) req.query = query
165
+ // A repeated name, or an order an object cannot hold, is only kept by the pairs.
166
+ if (!samePairs(pairs, Object.entries(query))) req.queryPairs = pairs
132
167
  if (headers !== undefined) req.headers = headers
133
- if (body !== undefined) req.body = body
168
+ if (body !== undefined) {
169
+ req.body = body.body
170
+ if (body.text) req.bodyEncoding = "text"
171
+ }
134
172
  } else {
135
173
  req.method = request.method ?? "GET"
136
174
  req.path = request.path
137
175
  if (request.query !== undefined) req.query = request.query
176
+ if (request.queryPairs !== undefined) req.queryPairs = request.queryPairs
138
177
  if (request.headers !== undefined) req.headers = request.headers
139
178
  if (request.body !== undefined) req.body = request.body
179
+ if (request.bodyEncoding !== undefined) req.bodyEncoding = request.bodyEncoding
140
180
  }
141
181
 
142
182
  let status: number
@@ -163,14 +203,22 @@ export async function captureIncident(
163
203
 
164
204
  function buildRequest(capsule: IncidentCapsule, origin: string): Request {
165
205
  const url = new URL(capsule.request.path, origin)
166
- for (const [key, value] of Object.entries(capsule.request.query ?? {})) {
167
- url.searchParams.set(key, value)
206
+ if (capsule.request.queryPairs !== undefined) {
207
+ for (const [key, value] of capsule.request.queryPairs) url.searchParams.append(key, value)
208
+ } else {
209
+ for (const [key, value] of Object.entries(capsule.request.query ?? {})) {
210
+ url.searchParams.set(key, value)
211
+ }
168
212
  }
169
213
  const headers = new Headers(capsule.request.headers)
170
- const hasBody = capsule.request.body !== undefined
171
- if (hasBody && !headers.has("content-type")) headers.set("content-type", "application/json")
214
+ const body = capsule.request.body
172
215
  const init: RequestInit = { method: capsule.request.method, headers }
173
- if (hasBody) init.body = JSON.stringify(capsule.request.body)
216
+ if (capsule.request.bodyEncoding === "text" && typeof body === "string") {
217
+ init.body = body
218
+ } else if (body !== undefined) {
219
+ if (!headers.has("content-type")) headers.set("content-type", "application/json")
220
+ init.body = JSON.stringify(body)
221
+ }
174
222
  return new Request(url.toString(), init)
175
223
  }
176
224
 
@@ -233,9 +281,9 @@ export async function assertIncidentReplays(
233
281
  const REDACTED = "<redacted>"
234
282
 
235
283
  /**
236
- * Redact leaf string values by default (unless the dotted key path is allow-listed). Non-strings are
237
- * kept - they carry the structure that makes the fixture reproduce - so review the emitted file. This
238
- * is intentionally aggressive: a committed fixture must not leak PII/secrets.
284
+ * Redact leaf string values by default (unless the dotted key path is allow-listed). Non-strings and
285
+ * object keys are kept - they carry the structure that makes the fixture reproduce - and the emitted
286
+ * file says so in a review banner. A committed fixture must not leak PII/secrets.
239
287
  */
240
288
  export function redactForEmission(value: unknown, allow: ReadonlySet<string>, path = ""): unknown {
241
289
  if (typeof value === "string") return allow.has(path) ? value : REDACTED
@@ -251,6 +299,14 @@ export function redactForEmission(value: unknown, allow: ReadonlySet<string>, pa
251
299
  return value
252
300
  }
253
301
 
302
+ /** Whether a redacted body still holds request data verbatim: a number, a boolean or an object key. */
303
+ function keepsVerbatim(value: unknown): boolean {
304
+ if (typeof value === "number" || typeof value === "boolean") return true
305
+ if (Array.isArray(value)) return value.some(keepsVerbatim)
306
+ if (value !== null && typeof value === "object") return Object.keys(value).length > 0
307
+ return false
308
+ }
309
+
254
310
  export interface GenerateRegressionTestOptions {
255
311
  /** Import specifier for the app under test. Default `"../src/app"`. */
256
312
  readonly importPath?: string
@@ -260,12 +316,33 @@ export interface GenerateRegressionTestOptions {
260
316
  readonly assertShape?: boolean
261
317
  /** Dotted key paths in the request body whose string values are safe to embed verbatim. */
262
318
  readonly allow?: readonly string[]
319
+ /** Query parameter names whose values are safe to embed verbatim. Default none. */
320
+ readonly allowQuery?: readonly string[]
321
+ /** Header names (any case) whose values are safe to embed verbatim. Default `["content-type"]`. */
322
+ readonly allowHeaders?: readonly string[]
323
+ /** Embed the capsule's `note` verbatim. Default false: a note is free text and is redacted. */
324
+ readonly keepNote?: boolean
325
+ }
326
+
327
+ function redactRecord(
328
+ record: Readonly<Record<string, string>> | undefined,
329
+ allowed: ReadonlySet<string>,
330
+ normalize: (name: string) => string,
331
+ ): Record<string, string> | undefined {
332
+ if (record === undefined) return undefined
333
+ const out: Record<string, string> = {}
334
+ for (const [name, value] of Object.entries(record)) {
335
+ out[name] = allowed.has(normalize(name)) ? value : REDACTED
336
+ }
337
+ return out
263
338
  }
264
339
 
265
340
  /**
266
- * Emit a committable regression test from a capsule. Request string values are redacted BY DEFAULT with
267
- * a sanitize banner - replace the `<redacted>` placeholders with safe, reproducing values before you
268
- * commit. The test asserts the response contract via {@link assertIncidentReplays}.
341
+ * Emit a committable regression test from a capsule. Body strings, query values, header values (bar
342
+ * `content-type`) and the note are redacted BY DEFAULT with a sanitize banner - replace the
343
+ * `<redacted>` placeholders with safe, reproducing values before you commit. The path is kept
344
+ * verbatim because it selects the route, so the file always asks for it to be reviewed. The test
345
+ * asserts the response contract via {@link assertIncidentReplays}.
269
346
  */
270
347
  export function generateRegressionTest(
271
348
  capsule: IncidentCapsule,
@@ -273,26 +350,50 @@ export function generateRegressionTest(
273
350
  ): string {
274
351
  const importPath = options.importPath ?? "../src/app"
275
352
  const appExport = options.appExport ?? "app"
353
+ if (!/^[A-Za-z_$][\w$]*$/.test(appExport))
354
+ throw new TypeError(
355
+ `generateRegressionTest: appExport must be an identifier, got ${JSON.stringify(appExport)}`,
356
+ )
276
357
  const allow = new Set(options.allow ?? [])
358
+ const allowQuery = new Set(options.allowQuery ?? [])
359
+ const query = redactRecord(capsule.request.query, allowQuery, (n) => n)
360
+ const queryPairs = capsule.request.queryPairs?.map(([name, value]): readonly [string, string] => [
361
+ name,
362
+ allowQuery.has(name) ? value : REDACTED,
363
+ ])
364
+ const headers = redactRecord(
365
+ capsule.request.headers,
366
+ new Set((options.allowHeaders ?? ["content-type"]).map((name) => name.toLowerCase())),
367
+ (name) => name.toLowerCase(),
368
+ )
277
369
  const safeRequest: CapturedRequest = {
278
370
  ...capsule.request,
371
+ ...(query !== undefined ? { query } : {}),
372
+ ...(queryPairs !== undefined ? { queryPairs } : {}),
373
+ ...(headers !== undefined ? { headers } : {}),
279
374
  ...(capsule.request.body !== undefined
280
375
  ? { body: redactForEmission(capsule.request.body, allow) }
281
376
  : {}),
282
377
  }
283
- const emitted: IncidentCapsule = { ...capsule, request: safeRequest }
378
+ const emitted: IncidentCapsule = {
379
+ ...capsule,
380
+ request: safeRequest,
381
+ ...(capsule.note !== undefined && options.keepNote !== true ? { note: REDACTED } : {}),
382
+ }
284
383
  const redacted = JSON.stringify(emitted, null, 2).includes(REDACTED)
384
+ const verbatim = keepsVerbatim(safeRequest.body)
285
385
 
286
386
  return `// GENERATED by nifra incident → regression.
287
- ${redacted ? '// ⚠️ SANITIZE BEFORE COMMITTING: replace every "<redacted>" with a safe, non-PII value that\n// still reproduces the response. A committed fixture must not carry PII or secrets.\n' : ""}// Asserts the response CONTRACT (status${options.assertShape ? " + shape" : ""}) still reproduces.
387
+ ${redacted ? '// ⚠️ SANITIZE BEFORE COMMITTING: replace every "<redacted>" with a safe, non-PII value that\n// still reproduces the response. A committed fixture must not carry PII or secrets.\n' : ""}${verbatim ? "// ⚠️ REVIEW BEFORE COMMITTING: numbers, booleans and object keys in the request body are kept\n// verbatim, since they shape the request. Replace any that carry PII or secrets.\n" : ""}// Review the request path before committing: it is kept verbatim because it selects the route.
388
+ // Asserts the response CONTRACT (status${options.assertShape ? " + shape" : ""}) still reproduces.
288
389
  import { describe, it } from "bun:test"
289
390
  import { assertIncidentReplays, type IncidentCapsule } from "@nifrajs/testing"
290
- import { ${appExport} } from "${importPath}"
391
+ import { ${appExport} } from ${JSON.stringify(importPath)}
291
392
 
292
393
  const capsule: IncidentCapsule = ${JSON.stringify(emitted, null, 2)}
293
394
 
294
- describe("regression: ${capsule.request.method} ${capsule.request.path}", () => {
295
- it("reproduces status ${capsule.response.status}", async () => {
395
+ describe(${JSON.stringify(`regression: ${capsule.request.method} ${capsule.request.path}`)}, () => {
396
+ it(${JSON.stringify(`reproduces status ${capsule.response.status}`)}, async () => {
296
397
  await assertIncidentReplays(${appExport}, capsule, { assertShape: ${options.assertShape === true} })
297
398
  })
298
399
  })
package/src/index.ts CHANGED
@@ -49,6 +49,7 @@ export {
49
49
  type AgentFailureMatrixCase,
50
50
  AgentFailureMatrixError,
51
51
  type AgentFailureMatrixKind,
52
+ type AgentFailureMatrixOptions,
52
53
  type AgentFailureMatrixReport,
53
54
  assertAgentEval,
54
55
  assertAgentEvalBaseline,