@nifrajs/testing 4.0.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -4
- package/dist/adversarial.d.ts +6 -38
- package/dist/adversarial.d.ts.map +1 -1
- package/dist/adversarial.js +304 -83
- package/dist/adversarial.js.map +1 -1
- package/dist/agent-eval.d.ts +25 -8
- package/dist/agent-eval.d.ts.map +1 -1
- package/dist/agent-eval.js +11 -6
- package/dist/agent-eval.js.map +1 -1
- package/dist/certification.d.ts +11 -1
- package/dist/certification.d.ts.map +1 -1
- package/dist/certification.js +136 -49
- package/dist/certification.js.map +1 -1
- package/dist/cookies.d.ts +1 -1
- package/dist/cookies.d.ts.map +1 -1
- package/dist/cookies.js +5 -3
- package/dist/cookies.js.map +1 -1
- package/dist/data-conformance.d.ts.map +1 -1
- package/dist/data-conformance.js +20 -8
- package/dist/data-conformance.js.map +1 -1
- package/dist/incident.d.ts +21 -6
- package/dist/incident.d.ts.map +1 -1
- package/dist/incident.js +99 -23
- package/dist/incident.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/trajectory.d.ts.map +1 -1
- package/dist/trajectory.js +31 -21
- package/dist/trajectory.js.map +1 -1
- package/dist/zod.d.ts +2 -2
- package/package.json +7 -7
- package/src/adversarial.ts +378 -147
- package/src/agent-eval.ts +30 -8
- package/src/certification.ts +137 -45
- package/src/cookies.ts +4 -2
- package/src/data-conformance.ts +23 -7
- package/src/incident.ts +121 -20
- package/src/index.ts +1 -0
- package/src/trajectory.ts +28 -28
- package/src/zod.ts +2 -2
package/src/agent-eval.ts
CHANGED
|
@@ -106,7 +106,10 @@ export type ComparisonCode =
|
|
|
106
106
|
| "improved"
|
|
107
107
|
| "tolerated"
|
|
108
108
|
| "regressed"
|
|
109
|
+
/** The baseline has a verdict the current report no longer produces. */
|
|
109
110
|
| "missing"
|
|
111
|
+
/** The current report has a verdict the baseline lacks. */
|
|
112
|
+
| "added"
|
|
110
113
|
| "incomparable"
|
|
111
114
|
|
|
112
115
|
/** Explicit tolerance for a score regression. `rubricId` omitted applies to every rubric. */
|
|
@@ -120,7 +123,10 @@ export interface ScoreTolerance {
|
|
|
120
123
|
|
|
121
124
|
export interface BaselineOptions {
|
|
122
125
|
readonly tolerances?: readonly ScoreTolerance[]
|
|
123
|
-
/**
|
|
126
|
+
/**
|
|
127
|
+
* Comparison codes that fail the assertion. Default `["regressed", "missing", "incomparable"]`:
|
|
128
|
+
* a verdict that disappeared or whose rubric changed shape cannot show it did not regress.
|
|
129
|
+
*/
|
|
124
130
|
readonly failOn?: readonly ComparisonCode[]
|
|
125
131
|
}
|
|
126
132
|
|
|
@@ -207,13 +213,26 @@ function matrixSchedule(caseId: string): readonly FailureDirective[] {
|
|
|
207
213
|
return [{ point: `agent.${caseId}`, kind: "crash" }]
|
|
208
214
|
}
|
|
209
215
|
|
|
216
|
+
export interface AgentFailureMatrixOptions {
|
|
217
|
+
readonly seed?: number
|
|
218
|
+
/**
|
|
219
|
+
* Drive the agent under test across one boundary, calling `checkpoint()` where that boundary's
|
|
220
|
+
* fault lands. The case passes only when the injected crash surfaces from `execute`. Without it
|
|
221
|
+
* each case runs a bare fixture, which proves the schedule and replay identity, not an agent.
|
|
222
|
+
*/
|
|
223
|
+
readonly execute?: (
|
|
224
|
+
boundary: { readonly id: string; readonly kind: AgentFailureMatrixKind },
|
|
225
|
+
checkpoint: () => void,
|
|
226
|
+
) => unknown
|
|
227
|
+
}
|
|
228
|
+
|
|
210
229
|
/**
|
|
211
|
-
*
|
|
212
|
-
*
|
|
213
|
-
*
|
|
230
|
+
* Inject a crash at each agent boundary. The implementation intentionally delegates scheduling and
|
|
231
|
+
* replay identity to the existing failure laboratory; this is a matrix consumer, not a second fault
|
|
232
|
+
* engine.
|
|
214
233
|
*/
|
|
215
234
|
export async function runAgentFailureMatrix(
|
|
216
|
-
options:
|
|
235
|
+
options: AgentFailureMatrixOptions = {},
|
|
217
236
|
): Promise<AgentFailureMatrixReport> {
|
|
218
237
|
const seed = Number.isFinite(options.seed) ? Math.trunc(options.seed as number) >>> 0 : 0x4e494652
|
|
219
238
|
const cases: AgentFailureMatrixCase[] = []
|
|
@@ -223,7 +242,9 @@ export async function runAgentFailureMatrix(
|
|
|
223
242
|
{
|
|
224
243
|
name: `agent-${caseId}`,
|
|
225
244
|
execute(lab) {
|
|
226
|
-
lab.checkpoint(`agent.${caseId}`)
|
|
245
|
+
const checkpoint = (): void => lab.checkpoint(`agent.${caseId}`)
|
|
246
|
+
if (options.execute !== undefined) return options.execute(entry, checkpoint)
|
|
247
|
+
checkpoint()
|
|
227
248
|
return true
|
|
228
249
|
},
|
|
229
250
|
verify: ({ error }) => error instanceof FailureInjectedError,
|
|
@@ -590,7 +611,7 @@ export async function compareAgentEvalBaseline(
|
|
|
590
611
|
}),
|
|
591
612
|
)
|
|
592
613
|
}
|
|
593
|
-
const failOn = new Set(options.failOn ?? ["regressed"])
|
|
614
|
+
const failOn = new Set(options.failOn ?? ["regressed", "missing", "incomparable"])
|
|
594
615
|
const regressions = comparisons
|
|
595
616
|
.filter((entry) => failOn.has(entry.code))
|
|
596
617
|
.map((entry) => entry.regressionId)
|
|
@@ -629,7 +650,8 @@ function classify(
|
|
|
629
650
|
rubricId: string,
|
|
630
651
|
options: BaselineOptions,
|
|
631
652
|
): ComparisonCode {
|
|
632
|
-
if (base === undefined
|
|
653
|
+
if (base === undefined) return "added"
|
|
654
|
+
if (cur === undefined) return "missing"
|
|
633
655
|
if (baseShape === undefined || curShape === undefined || baseShape.digest !== curShape.digest)
|
|
634
656
|
return "incomparable"
|
|
635
657
|
const baseRank = curShape.outcomes.indexOf(base.outcome)
|
package/src/certification.ts
CHANGED
|
@@ -252,57 +252,99 @@ export interface CertifiableCacheStore {
|
|
|
252
252
|
delete(key: string): void | Promise<void>
|
|
253
253
|
invalidateTag(tag: string): void | Promise<void>
|
|
254
254
|
clear(): void | Promise<void>
|
|
255
|
+
fence?(key: string, tags: readonly string[]): unknown | Promise<unknown>
|
|
256
|
+
setIfFence?(
|
|
257
|
+
key: string,
|
|
258
|
+
entry: CertifiableCacheEntry,
|
|
259
|
+
tags: readonly string[],
|
|
260
|
+
token: unknown,
|
|
261
|
+
): boolean | Promise<boolean>
|
|
255
262
|
}
|
|
256
263
|
|
|
257
|
-
|
|
264
|
+
/**
|
|
265
|
+
* The cache-store profile. `fencing: true` adds the optional `fenced-write` capability: `setIfFence`
|
|
266
|
+
* refuses a write after an invalidation of its tag, a delete or set of its key, or a clear since its
|
|
267
|
+
* `fence`, and accepts one after an unrelated change. A store several caches share needs it, or a load
|
|
268
|
+
* running in one cache can store a value another cache just invalidated.
|
|
269
|
+
*/
|
|
270
|
+
export function cacheStoreCertificationProfile(
|
|
271
|
+
options: { readonly fencing?: boolean } = {},
|
|
272
|
+
): AdapterCertificationProfile<CertifiableCacheStore> {
|
|
273
|
+
// Fresh on any clock until 2100: a store that honors expiry on the wall clock must still keep it.
|
|
258
274
|
const entry = (value: unknown): CertifiableCacheEntry => ({
|
|
259
275
|
value,
|
|
260
|
-
staleAt:
|
|
261
|
-
expiresAt:
|
|
276
|
+
staleAt: 4_102_444_800_000,
|
|
277
|
+
expiresAt: 4_102_448_400_000,
|
|
262
278
|
})
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
{
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
if (JSON.stringify(found?.value) !== '{"answer":42}') throw new Error("CacheRoundtrip")
|
|
276
|
-
await store.delete(key)
|
|
277
|
-
if ((await store.get(key)) !== undefined) throw new Error("CacheDelete")
|
|
278
|
-
},
|
|
279
|
+
const capabilities = ["roundtrip", "tag-invalidation", "clear"]
|
|
280
|
+
const checks: CertificationCheck<CertifiableCacheStore>[] = [
|
|
281
|
+
{
|
|
282
|
+
id: "set-get-delete",
|
|
283
|
+
capability: "roundtrip",
|
|
284
|
+
async run(store, context) {
|
|
285
|
+
const key = context.key("roundtrip")
|
|
286
|
+
await store.set(key, entry({ answer: 42 }), [])
|
|
287
|
+
const found = await store.get(key)
|
|
288
|
+
if (JSON.stringify(found?.value) !== '{"answer":42}') throw new Error("CacheRoundtrip")
|
|
289
|
+
await store.delete(key)
|
|
290
|
+
if ((await store.get(key)) !== undefined) throw new Error("CacheDelete")
|
|
279
291
|
},
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
id: "invalidate-tag",
|
|
295
|
+
capability: "tag-invalidation",
|
|
296
|
+
async run(store, context) {
|
|
297
|
+
const tagged = context.key("tagged")
|
|
298
|
+
const other = context.key("other")
|
|
299
|
+
const tag = context.key("tag")
|
|
300
|
+
await store.set(tagged, entry(1), [tag])
|
|
301
|
+
await store.set(other, entry(2), [context.key("other-tag")])
|
|
302
|
+
await store.invalidateTag(tag)
|
|
303
|
+
if ((await store.get(tagged)) !== undefined || (await store.get(other))?.value !== 2)
|
|
304
|
+
throw new Error("CacheTagInvalidation")
|
|
293
305
|
},
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
id: "clear",
|
|
309
|
+
capability: "clear",
|
|
310
|
+
async run(store, context) {
|
|
311
|
+
const key = context.key("clear")
|
|
312
|
+
await store.set(key, entry(1), [])
|
|
313
|
+
await store.clear()
|
|
314
|
+
if ((await store.get(key)) !== undefined) throw new Error("CacheClear")
|
|
303
315
|
},
|
|
304
|
-
|
|
316
|
+
},
|
|
317
|
+
]
|
|
318
|
+
if (options.fencing === true) {
|
|
319
|
+
capabilities.push("fenced-write")
|
|
320
|
+
checks.push({
|
|
321
|
+
id: "fenced-write",
|
|
322
|
+
capability: "fenced-write",
|
|
323
|
+
async run(store, context) {
|
|
324
|
+
if (store.fence === undefined || store.setIfFence === undefined)
|
|
325
|
+
throw new Error("CacheFenceMissing")
|
|
326
|
+
const key = context.key("fenced")
|
|
327
|
+
const tag = context.key("fenced-tag")
|
|
328
|
+
const changes: readonly [() => void | Promise<void>, boolean][] = [
|
|
329
|
+
[() => store.set(context.key("unrelated"), entry(0), []), true],
|
|
330
|
+
[() => store.invalidateTag(tag), false],
|
|
331
|
+
[() => store.delete(key), false],
|
|
332
|
+
[() => store.set(key, entry("newer"), []), false],
|
|
333
|
+
[() => store.clear(), false],
|
|
334
|
+
]
|
|
335
|
+
for (const [change, writes] of changes) {
|
|
336
|
+
await store.delete(key)
|
|
337
|
+
const token = await store.fence(key, [tag])
|
|
338
|
+
await change()
|
|
339
|
+
if ((await store.setIfFence(key, entry("loaded"), [tag], token)) !== writes)
|
|
340
|
+
throw new Error("CacheFencedWrite")
|
|
341
|
+
if (((await store.get(key))?.value === "loaded") !== writes)
|
|
342
|
+
throw new Error("CacheFencedWriteStored")
|
|
343
|
+
}
|
|
344
|
+
},
|
|
345
|
+
})
|
|
305
346
|
}
|
|
347
|
+
return { id: "cache-store", version: 2, capabilities, checks }
|
|
306
348
|
}
|
|
307
349
|
|
|
308
350
|
export interface CertifiableStoredJob {
|
|
@@ -367,6 +409,19 @@ export function jobStoreCertificationProfile(
|
|
|
367
409
|
if ((await store.counts()).pending !== 0) throw new Error("JobComplete")
|
|
368
410
|
},
|
|
369
411
|
},
|
|
412
|
+
{
|
|
413
|
+
// A lease hides the job from every other worker until it runs out, then hands it back.
|
|
414
|
+
id: "lease-exclusive",
|
|
415
|
+
capability: "lease-complete",
|
|
416
|
+
async run(store) {
|
|
417
|
+
const id = await store.enqueue({ name: "cert", payload: null, runAt: 0, maxAttempts: 3 })
|
|
418
|
+
const first = await store.lease(0, 1, 100)
|
|
419
|
+
if (first.length !== 1 || first[0]?.id !== id) throw new Error("JobLease")
|
|
420
|
+
if ((await store.lease(50, 1, 100)).length !== 0) throw new Error("JobLeaseShared")
|
|
421
|
+
const expired = await store.lease(101, 1, 100)
|
|
422
|
+
if (expired.length !== 1 || expired[0]?.id !== id) throw new Error("JobLeaseExpiry")
|
|
423
|
+
},
|
|
424
|
+
},
|
|
370
425
|
{
|
|
371
426
|
id: "retry-schedule",
|
|
372
427
|
capability: "retry-schedule",
|
|
@@ -422,7 +477,7 @@ export function jobStoreCertificationProfile(
|
|
|
422
477
|
},
|
|
423
478
|
})
|
|
424
479
|
}
|
|
425
|
-
return { id: "job-store", version:
|
|
480
|
+
return { id: "job-store", version: 2, capabilities, checks }
|
|
426
481
|
}
|
|
427
482
|
|
|
428
483
|
export interface CertifiableStorageObject {
|
|
@@ -463,10 +518,24 @@ export interface CertifiableStorageAdapter {
|
|
|
463
518
|
move?(sourceKey: string, destinationKey: string): Promise<void>
|
|
464
519
|
}
|
|
465
520
|
|
|
521
|
+
/** Keys the storage key contract refuses: a `..` segment anywhere, an absolute path, a backslash, a
|
|
522
|
+
* NUL, and an empty, `.` or trailing segment. Refusing only `../escape` leaves the rest open. */
|
|
523
|
+
const UNSAFE_STORAGE_KEYS = [
|
|
524
|
+
"../escape",
|
|
525
|
+
"cert/../../escape",
|
|
526
|
+
"/cert/absolute",
|
|
527
|
+
"cert\\..\\escape",
|
|
528
|
+
"cert/nul\0",
|
|
529
|
+
"",
|
|
530
|
+
"cert//empty-segment",
|
|
531
|
+
"cert/./dot-segment",
|
|
532
|
+
"cert/trailing/",
|
|
533
|
+
]
|
|
534
|
+
|
|
466
535
|
export function storageAdapterCertificationProfile(
|
|
467
536
|
options: { readonly paging?: boolean; readonly presign?: boolean; readonly move?: boolean } = {},
|
|
468
537
|
): AdapterCertificationProfile<CertifiableStorageAdapter> {
|
|
469
|
-
const capabilities = ["object-roundtrip", "overwrite", "list-delete"]
|
|
538
|
+
const capabilities = ["object-roundtrip", "overwrite", "list-delete", "key-safety"]
|
|
470
539
|
const checks: CertificationCheck<CertifiableStorageAdapter>[] = [
|
|
471
540
|
{
|
|
472
541
|
id: "object-roundtrip",
|
|
@@ -513,6 +582,29 @@ export function storageAdapterCertificationProfile(
|
|
|
513
582
|
throw new Error("StorageDelete")
|
|
514
583
|
},
|
|
515
584
|
},
|
|
585
|
+
{
|
|
586
|
+
id: "unsafe-keys",
|
|
587
|
+
capability: "key-safety",
|
|
588
|
+
async run(store) {
|
|
589
|
+
const refuses = async (call: () => Promise<unknown>): Promise<boolean> => {
|
|
590
|
+
try {
|
|
591
|
+
await call()
|
|
592
|
+
return false
|
|
593
|
+
} catch {
|
|
594
|
+
return true
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
for (const key of UNSAFE_STORAGE_KEYS) {
|
|
598
|
+
const refused = await Promise.all([
|
|
599
|
+
refuses(() => store.put(key, "x")),
|
|
600
|
+
refuses(() => store.get(key)),
|
|
601
|
+
refuses(() => store.exists(key)),
|
|
602
|
+
refuses(() => store.delete(key)),
|
|
603
|
+
])
|
|
604
|
+
if (refused.includes(false)) throw new Error("StorageUnsafeKey")
|
|
605
|
+
}
|
|
606
|
+
},
|
|
607
|
+
},
|
|
516
608
|
]
|
|
517
609
|
if (options.paging === true) {
|
|
518
610
|
capabilities.push("cursor-paging")
|
|
@@ -576,7 +668,7 @@ export function storageAdapterCertificationProfile(
|
|
|
576
668
|
}
|
|
577
669
|
return {
|
|
578
670
|
id: "storage-adapter",
|
|
579
|
-
version:
|
|
671
|
+
version: 2,
|
|
580
672
|
capabilities,
|
|
581
673
|
checks,
|
|
582
674
|
}
|
package/src/cookies.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A tiny cookie jar for in-process tests - parses `Set-Cookie` off responses and emits a `Cookie`
|
|
3
3
|
* request header, so a login → authenticated-request flow works without threading headers by hand. It
|
|
4
|
-
* honours removal (`Max-Age
|
|
4
|
+
* honours removal (`Max-Age` of zero or less, or a past `Expires`) so logout clears the cookie; other attributes
|
|
5
5
|
* (Domain/Path/Secure/SameSite) are ignored - everything is same-origin in-process.
|
|
6
6
|
*/
|
|
7
7
|
|
|
@@ -27,7 +27,9 @@ function setCookies(headers: Headers): string[] {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
function isRemoval(attributes: string): boolean {
|
|
30
|
-
|
|
30
|
+
// RFC 6265 5.2.2: any Max-Age of zero or less removes the cookie, and Max-Age outranks Expires.
|
|
31
|
+
const maxAge = /(?:^|;)\s*max-age=(-?\d+)\s*(?:;|$)/.exec(attributes)
|
|
32
|
+
if (maxAge !== null) return Number(maxAge[1]) <= 0
|
|
31
33
|
const expires = /expires=([^;]+)/.exec(attributes)
|
|
32
34
|
if (expires === null) return false
|
|
33
35
|
const at = Date.parse(expires[1] as string)
|
package/src/data-conformance.ts
CHANGED
|
@@ -215,19 +215,29 @@ export async function runDataAdapterConformance<Adapter, Row, Snapshot, Cursor =
|
|
|
215
215
|
}
|
|
216
216
|
})
|
|
217
217
|
|
|
218
|
+
const keysIn = async (scope: DataConformanceScope): Promise<ReadonlySet<string>> =>
|
|
219
|
+
new Set((await target.read(primary, scope)).map(target.rowKey))
|
|
220
|
+
const sameKeys = (a: ReadonlySet<string>, b: ReadonlySet<string>): boolean =>
|
|
221
|
+
a.size === b.size && [...a].every((key) => b.has(key))
|
|
222
|
+
|
|
218
223
|
await run("cancellation", async () => {
|
|
219
224
|
const controller = new AbortController()
|
|
220
225
|
controller.abort()
|
|
226
|
+
let refused = false
|
|
221
227
|
try {
|
|
222
228
|
await target.write(primary, scopes.left, rows.rollback, controller.signal)
|
|
223
229
|
} catch (error) {
|
|
224
|
-
|
|
225
|
-
throw error
|
|
230
|
+
// Any other failure (a refused connection, a rejected row) proves nothing about the signal.
|
|
231
|
+
if (!isAbortLike(error) && error !== controller.signal.reason) throw error
|
|
232
|
+
refused = true
|
|
226
233
|
}
|
|
227
|
-
throw new Error("cancelled write resolved successfully")
|
|
234
|
+
if (!refused) throw new Error("cancelled write resolved successfully")
|
|
235
|
+
if ((await keysIn(scopes.left)).has(target.rowKey(rows.rollback)))
|
|
236
|
+
throw new Error("cancelled write reached durable state")
|
|
228
237
|
})
|
|
229
238
|
|
|
230
239
|
await run("pagination", async () => {
|
|
240
|
+
const expected = await keysIn(scopes.left)
|
|
231
241
|
const seen: string[] = []
|
|
232
242
|
const cursors = new Set<string>()
|
|
233
243
|
let cursor: Cursor | undefined
|
|
@@ -242,16 +252,22 @@ export async function runDataAdapterConformance<Adapter, Row, Snapshot, Cursor =
|
|
|
242
252
|
if (pageNumber === 99) throw new Error("pagination did not terminate")
|
|
243
253
|
}
|
|
244
254
|
if (new Set(seen).size !== seen.length) throw new Error("pagination returned a duplicate row")
|
|
255
|
+
if (!sameKeys(new Set(seen), expected))
|
|
256
|
+
throw new Error("pagination did not return exactly the scope's rows")
|
|
245
257
|
})
|
|
246
258
|
|
|
247
259
|
await run("backup-restore", async () => {
|
|
260
|
+
const left = await keysIn(scopes.left)
|
|
261
|
+
const right = await keysIn(scopes.right)
|
|
248
262
|
const snapshot = await target.snapshot(primary)
|
|
249
263
|
await target.seed(primary, scopes.left, rows.rollback)
|
|
250
264
|
await target.restore(primary, snapshot)
|
|
251
|
-
|
|
252
|
-
if (
|
|
253
|
-
|
|
254
|
-
|
|
265
|
+
// Restore returns every scope to the snapshot: no later row, and no row the snapshot held lost.
|
|
266
|
+
if (
|
|
267
|
+
!sameKeys(await keysIn(scopes.left), left) ||
|
|
268
|
+
!sameKeys(await keysIn(scopes.right), right)
|
|
269
|
+
)
|
|
270
|
+
throw new Error("restore did not return the snapshot's rows")
|
|
255
271
|
})
|
|
256
272
|
|
|
257
273
|
await run("multi-instance", async () => {
|
package/src/incident.ts
CHANGED
|
@@ -18,16 +18,23 @@ export interface CapturedRequestInput {
|
|
|
18
18
|
readonly method?: string
|
|
19
19
|
readonly path: string
|
|
20
20
|
readonly query?: Record<string, string>
|
|
21
|
+
readonly queryPairs?: readonly (readonly [string, string])[]
|
|
21
22
|
readonly headers?: Record<string, string>
|
|
22
23
|
readonly body?: unknown
|
|
24
|
+
readonly bodyEncoding?: "text"
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
export interface CapturedRequest {
|
|
26
28
|
readonly method: string
|
|
27
29
|
readonly path: string
|
|
30
|
+
/** The query by name; a repeated name keeps its last value. */
|
|
28
31
|
readonly query?: Record<string, string>
|
|
32
|
+
/** Every query pair in order, present when `query` cannot rebuild it. Replay prefers it. */
|
|
33
|
+
readonly queryPairs?: readonly (readonly [string, string])[]
|
|
29
34
|
readonly headers?: Record<string, string>
|
|
30
35
|
readonly body?: unknown
|
|
36
|
+
/** `"text"`: `body` is the request text as sent, and replay sends it verbatim. Absent: JSON. */
|
|
37
|
+
readonly bodyEncoding?: "text"
|
|
31
38
|
}
|
|
32
39
|
|
|
33
40
|
export interface IncidentCapsule {
|
|
@@ -95,6 +102,29 @@ async function readBody(source: Request | Response): Promise<unknown> {
|
|
|
95
102
|
return text
|
|
96
103
|
}
|
|
97
104
|
|
|
105
|
+
/** The request body as captured: parsed JSON when the request said so, its text otherwise. */
|
|
106
|
+
async function readRequestBody(
|
|
107
|
+
request: Request,
|
|
108
|
+
): Promise<{ readonly body: unknown; readonly text: boolean } | undefined> {
|
|
109
|
+
const text = await request.clone().text()
|
|
110
|
+
if (text === "") return undefined
|
|
111
|
+
if ((request.headers.get("content-type") ?? "").includes("application/json")) {
|
|
112
|
+
try {
|
|
113
|
+
return { body: JSON.parse(text), text: false }
|
|
114
|
+
} catch {}
|
|
115
|
+
}
|
|
116
|
+
return { body: text, text: true }
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function samePairs(
|
|
120
|
+
a: readonly (readonly [string, string])[],
|
|
121
|
+
b: readonly (readonly [string, string])[],
|
|
122
|
+
): boolean {
|
|
123
|
+
return (
|
|
124
|
+
a.length === b.length && a.every(([key, value], i) => b[i]?.[0] === key && b[i]?.[1] === value)
|
|
125
|
+
)
|
|
126
|
+
}
|
|
127
|
+
|
|
98
128
|
function pickHeaders(
|
|
99
129
|
headers: Headers,
|
|
100
130
|
allow: readonly string[],
|
|
@@ -118,25 +148,35 @@ export async function captureIncident(
|
|
|
118
148
|
method: string
|
|
119
149
|
path: string
|
|
120
150
|
query?: Record<string, string>
|
|
151
|
+
queryPairs?: readonly (readonly [string, string])[]
|
|
121
152
|
headers?: Record<string, string>
|
|
122
153
|
body?: unknown
|
|
154
|
+
bodyEncoding?: "text"
|
|
123
155
|
} = { method: "GET", path: "/" }
|
|
124
156
|
if (request instanceof Request) {
|
|
125
157
|
const url = new URL(request.url)
|
|
126
|
-
const
|
|
158
|
+
const pairs = [...url.searchParams.entries()]
|
|
159
|
+
const query = Object.fromEntries(pairs)
|
|
127
160
|
const headers = pickHeaders(request.headers, allow)
|
|
128
|
-
const body = await
|
|
161
|
+
const body = await readRequestBody(request)
|
|
129
162
|
req.method = request.method
|
|
130
163
|
req.path = url.pathname
|
|
131
|
-
if (
|
|
164
|
+
if (pairs.length > 0) req.query = query
|
|
165
|
+
// A repeated name, or an order an object cannot hold, is only kept by the pairs.
|
|
166
|
+
if (!samePairs(pairs, Object.entries(query))) req.queryPairs = pairs
|
|
132
167
|
if (headers !== undefined) req.headers = headers
|
|
133
|
-
if (body !== undefined)
|
|
168
|
+
if (body !== undefined) {
|
|
169
|
+
req.body = body.body
|
|
170
|
+
if (body.text) req.bodyEncoding = "text"
|
|
171
|
+
}
|
|
134
172
|
} else {
|
|
135
173
|
req.method = request.method ?? "GET"
|
|
136
174
|
req.path = request.path
|
|
137
175
|
if (request.query !== undefined) req.query = request.query
|
|
176
|
+
if (request.queryPairs !== undefined) req.queryPairs = request.queryPairs
|
|
138
177
|
if (request.headers !== undefined) req.headers = request.headers
|
|
139
178
|
if (request.body !== undefined) req.body = request.body
|
|
179
|
+
if (request.bodyEncoding !== undefined) req.bodyEncoding = request.bodyEncoding
|
|
140
180
|
}
|
|
141
181
|
|
|
142
182
|
let status: number
|
|
@@ -163,14 +203,22 @@ export async function captureIncident(
|
|
|
163
203
|
|
|
164
204
|
function buildRequest(capsule: IncidentCapsule, origin: string): Request {
|
|
165
205
|
const url = new URL(capsule.request.path, origin)
|
|
166
|
-
|
|
167
|
-
url.searchParams.
|
|
206
|
+
if (capsule.request.queryPairs !== undefined) {
|
|
207
|
+
for (const [key, value] of capsule.request.queryPairs) url.searchParams.append(key, value)
|
|
208
|
+
} else {
|
|
209
|
+
for (const [key, value] of Object.entries(capsule.request.query ?? {})) {
|
|
210
|
+
url.searchParams.set(key, value)
|
|
211
|
+
}
|
|
168
212
|
}
|
|
169
213
|
const headers = new Headers(capsule.request.headers)
|
|
170
|
-
const
|
|
171
|
-
if (hasBody && !headers.has("content-type")) headers.set("content-type", "application/json")
|
|
214
|
+
const body = capsule.request.body
|
|
172
215
|
const init: RequestInit = { method: capsule.request.method, headers }
|
|
173
|
-
if (
|
|
216
|
+
if (capsule.request.bodyEncoding === "text" && typeof body === "string") {
|
|
217
|
+
init.body = body
|
|
218
|
+
} else if (body !== undefined) {
|
|
219
|
+
if (!headers.has("content-type")) headers.set("content-type", "application/json")
|
|
220
|
+
init.body = JSON.stringify(body)
|
|
221
|
+
}
|
|
174
222
|
return new Request(url.toString(), init)
|
|
175
223
|
}
|
|
176
224
|
|
|
@@ -233,9 +281,9 @@ export async function assertIncidentReplays(
|
|
|
233
281
|
const REDACTED = "<redacted>"
|
|
234
282
|
|
|
235
283
|
/**
|
|
236
|
-
* Redact leaf string values by default (unless the dotted key path is allow-listed). Non-strings
|
|
237
|
-
* kept - they carry the structure that makes the fixture reproduce -
|
|
238
|
-
*
|
|
284
|
+
* Redact leaf string values by default (unless the dotted key path is allow-listed). Non-strings and
|
|
285
|
+
* object keys are kept - they carry the structure that makes the fixture reproduce - and the emitted
|
|
286
|
+
* file says so in a review banner. A committed fixture must not leak PII/secrets.
|
|
239
287
|
*/
|
|
240
288
|
export function redactForEmission(value: unknown, allow: ReadonlySet<string>, path = ""): unknown {
|
|
241
289
|
if (typeof value === "string") return allow.has(path) ? value : REDACTED
|
|
@@ -251,6 +299,14 @@ export function redactForEmission(value: unknown, allow: ReadonlySet<string>, pa
|
|
|
251
299
|
return value
|
|
252
300
|
}
|
|
253
301
|
|
|
302
|
+
/** Whether a redacted body still holds request data verbatim: a number, a boolean or an object key. */
|
|
303
|
+
function keepsVerbatim(value: unknown): boolean {
|
|
304
|
+
if (typeof value === "number" || typeof value === "boolean") return true
|
|
305
|
+
if (Array.isArray(value)) return value.some(keepsVerbatim)
|
|
306
|
+
if (value !== null && typeof value === "object") return Object.keys(value).length > 0
|
|
307
|
+
return false
|
|
308
|
+
}
|
|
309
|
+
|
|
254
310
|
export interface GenerateRegressionTestOptions {
|
|
255
311
|
/** Import specifier for the app under test. Default `"../src/app"`. */
|
|
256
312
|
readonly importPath?: string
|
|
@@ -260,12 +316,33 @@ export interface GenerateRegressionTestOptions {
|
|
|
260
316
|
readonly assertShape?: boolean
|
|
261
317
|
/** Dotted key paths in the request body whose string values are safe to embed verbatim. */
|
|
262
318
|
readonly allow?: readonly string[]
|
|
319
|
+
/** Query parameter names whose values are safe to embed verbatim. Default none. */
|
|
320
|
+
readonly allowQuery?: readonly string[]
|
|
321
|
+
/** Header names (any case) whose values are safe to embed verbatim. Default `["content-type"]`. */
|
|
322
|
+
readonly allowHeaders?: readonly string[]
|
|
323
|
+
/** Embed the capsule's `note` verbatim. Default false: a note is free text and is redacted. */
|
|
324
|
+
readonly keepNote?: boolean
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function redactRecord(
|
|
328
|
+
record: Readonly<Record<string, string>> | undefined,
|
|
329
|
+
allowed: ReadonlySet<string>,
|
|
330
|
+
normalize: (name: string) => string,
|
|
331
|
+
): Record<string, string> | undefined {
|
|
332
|
+
if (record === undefined) return undefined
|
|
333
|
+
const out: Record<string, string> = {}
|
|
334
|
+
for (const [name, value] of Object.entries(record)) {
|
|
335
|
+
out[name] = allowed.has(normalize(name)) ? value : REDACTED
|
|
336
|
+
}
|
|
337
|
+
return out
|
|
263
338
|
}
|
|
264
339
|
|
|
265
340
|
/**
|
|
266
|
-
* Emit a committable regression test from a capsule.
|
|
267
|
-
*
|
|
268
|
-
*
|
|
341
|
+
* Emit a committable regression test from a capsule. Body strings, query values, header values (bar
|
|
342
|
+
* `content-type`) and the note are redacted BY DEFAULT with a sanitize banner - replace the
|
|
343
|
+
* `<redacted>` placeholders with safe, reproducing values before you commit. The path is kept
|
|
344
|
+
* verbatim because it selects the route, so the file always asks for it to be reviewed. The test
|
|
345
|
+
* asserts the response contract via {@link assertIncidentReplays}.
|
|
269
346
|
*/
|
|
270
347
|
export function generateRegressionTest(
|
|
271
348
|
capsule: IncidentCapsule,
|
|
@@ -273,26 +350,50 @@ export function generateRegressionTest(
|
|
|
273
350
|
): string {
|
|
274
351
|
const importPath = options.importPath ?? "../src/app"
|
|
275
352
|
const appExport = options.appExport ?? "app"
|
|
353
|
+
if (!/^[A-Za-z_$][\w$]*$/.test(appExport))
|
|
354
|
+
throw new TypeError(
|
|
355
|
+
`generateRegressionTest: appExport must be an identifier, got ${JSON.stringify(appExport)}`,
|
|
356
|
+
)
|
|
276
357
|
const allow = new Set(options.allow ?? [])
|
|
358
|
+
const allowQuery = new Set(options.allowQuery ?? [])
|
|
359
|
+
const query = redactRecord(capsule.request.query, allowQuery, (n) => n)
|
|
360
|
+
const queryPairs = capsule.request.queryPairs?.map(([name, value]): readonly [string, string] => [
|
|
361
|
+
name,
|
|
362
|
+
allowQuery.has(name) ? value : REDACTED,
|
|
363
|
+
])
|
|
364
|
+
const headers = redactRecord(
|
|
365
|
+
capsule.request.headers,
|
|
366
|
+
new Set((options.allowHeaders ?? ["content-type"]).map((name) => name.toLowerCase())),
|
|
367
|
+
(name) => name.toLowerCase(),
|
|
368
|
+
)
|
|
277
369
|
const safeRequest: CapturedRequest = {
|
|
278
370
|
...capsule.request,
|
|
371
|
+
...(query !== undefined ? { query } : {}),
|
|
372
|
+
...(queryPairs !== undefined ? { queryPairs } : {}),
|
|
373
|
+
...(headers !== undefined ? { headers } : {}),
|
|
279
374
|
...(capsule.request.body !== undefined
|
|
280
375
|
? { body: redactForEmission(capsule.request.body, allow) }
|
|
281
376
|
: {}),
|
|
282
377
|
}
|
|
283
|
-
const emitted: IncidentCapsule = {
|
|
378
|
+
const emitted: IncidentCapsule = {
|
|
379
|
+
...capsule,
|
|
380
|
+
request: safeRequest,
|
|
381
|
+
...(capsule.note !== undefined && options.keepNote !== true ? { note: REDACTED } : {}),
|
|
382
|
+
}
|
|
284
383
|
const redacted = JSON.stringify(emitted, null, 2).includes(REDACTED)
|
|
384
|
+
const verbatim = keepsVerbatim(safeRequest.body)
|
|
285
385
|
|
|
286
386
|
return `// GENERATED by nifra incident → regression.
|
|
287
|
-
${redacted ? '// ⚠️ SANITIZE BEFORE COMMITTING: replace every "<redacted>" with a safe, non-PII value that\n// still reproduces the response. A committed fixture must not carry PII or secrets.\n' : ""}//
|
|
387
|
+
${redacted ? '// ⚠️ SANITIZE BEFORE COMMITTING: replace every "<redacted>" with a safe, non-PII value that\n// still reproduces the response. A committed fixture must not carry PII or secrets.\n' : ""}${verbatim ? "// ⚠️ REVIEW BEFORE COMMITTING: numbers, booleans and object keys in the request body are kept\n// verbatim, since they shape the request. Replace any that carry PII or secrets.\n" : ""}// Review the request path before committing: it is kept verbatim because it selects the route.
|
|
388
|
+
// Asserts the response CONTRACT (status${options.assertShape ? " + shape" : ""}) still reproduces.
|
|
288
389
|
import { describe, it } from "bun:test"
|
|
289
390
|
import { assertIncidentReplays, type IncidentCapsule } from "@nifrajs/testing"
|
|
290
|
-
import { ${appExport} } from
|
|
391
|
+
import { ${appExport} } from ${JSON.stringify(importPath)}
|
|
291
392
|
|
|
292
393
|
const capsule: IncidentCapsule = ${JSON.stringify(emitted, null, 2)}
|
|
293
394
|
|
|
294
|
-
describe(
|
|
295
|
-
it(
|
|
395
|
+
describe(${JSON.stringify(`regression: ${capsule.request.method} ${capsule.request.path}`)}, () => {
|
|
396
|
+
it(${JSON.stringify(`reproduces status ${capsule.response.status}`)}, async () => {
|
|
296
397
|
await assertIncidentReplays(${appExport}, capsule, { assertShape: ${options.assertShape === true} })
|
|
297
398
|
})
|
|
298
399
|
})
|