@nifrajs/e2e 0.0.0-placeholder → 5.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/LICENSE +21 -0
  3. package/README.md +128 -0
  4. package/dist/agent.d.ts +119 -0
  5. package/dist/agent.d.ts.map +1 -0
  6. package/dist/agent.js +315 -0
  7. package/dist/agent.js.map +1 -0
  8. package/dist/cache.d.ts +84 -0
  9. package/dist/cache.d.ts.map +1 -0
  10. package/dist/cache.js +166 -0
  11. package/dist/cache.js.map +1 -0
  12. package/dist/decision.d.ts +54 -0
  13. package/dist/decision.d.ts.map +1 -0
  14. package/dist/decision.js +122 -0
  15. package/dist/decision.js.map +1 -0
  16. package/dist/engine.d.ts +75 -0
  17. package/dist/engine.d.ts.map +1 -0
  18. package/dist/engine.js +13 -0
  19. package/dist/engine.js.map +1 -0
  20. package/dist/evidence.d.ts +14 -0
  21. package/dist/evidence.d.ts.map +1 -0
  22. package/dist/evidence.js +52 -0
  23. package/dist/evidence.js.map +1 -0
  24. package/dist/expect.d.ts +28 -0
  25. package/dist/expect.d.ts.map +1 -0
  26. package/dist/expect.js +88 -0
  27. package/dist/expect.js.map +1 -0
  28. package/dist/fake.d.ts +35 -0
  29. package/dist/fake.d.ts.map +1 -0
  30. package/dist/fake.js +129 -0
  31. package/dist/fake.js.map +1 -0
  32. package/dist/index.d.ts +11 -0
  33. package/dist/index.d.ts.map +1 -0
  34. package/dist/index.js +8 -0
  35. package/dist/index.js.map +1 -0
  36. package/dist/internal/record.d.ts +3 -0
  37. package/dist/internal/record.d.ts.map +1 -0
  38. package/dist/internal/record.js +6 -0
  39. package/dist/internal/record.js.map +1 -0
  40. package/dist/locator.d.ts +53 -0
  41. package/dist/locator.d.ts.map +1 -0
  42. package/dist/locator.js +103 -0
  43. package/dist/locator.js.map +1 -0
  44. package/dist/params.d.ts +44 -0
  45. package/dist/params.d.ts.map +1 -0
  46. package/dist/params.js +108 -0
  47. package/dist/params.js.map +1 -0
  48. package/dist/playwright.d.ts +72 -0
  49. package/dist/playwright.d.ts.map +1 -0
  50. package/dist/playwright.js +169 -0
  51. package/dist/playwright.js.map +1 -0
  52. package/dist/replay.d.ts +34 -0
  53. package/dist/replay.d.ts.map +1 -0
  54. package/dist/replay.js +111 -0
  55. package/dist/replay.js.map +1 -0
  56. package/dist/route.d.ts +7 -0
  57. package/dist/route.d.ts.map +1 -0
  58. package/dist/route.js +26 -0
  59. package/dist/route.js.map +1 -0
  60. package/dist/session.d.ts +62 -0
  61. package/dist/session.d.ts.map +1 -0
  62. package/dist/session.js +105 -0
  63. package/dist/session.js.map +1 -0
  64. package/dist/snapshot.d.ts +8 -0
  65. package/dist/snapshot.d.ts.map +1 -0
  66. package/dist/snapshot.js +49 -0
  67. package/dist/snapshot.js.map +1 -0
  68. package/package.json +75 -1
  69. package/src/agent.ts +525 -0
  70. package/src/cache.ts +229 -0
  71. package/src/decision.ts +152 -0
  72. package/src/engine.ts +92 -0
  73. package/src/evidence.ts +58 -0
  74. package/src/expect.ts +171 -0
  75. package/src/fake.ts +185 -0
  76. package/src/index.ts +64 -0
  77. package/src/internal/record.ts +5 -0
  78. package/src/locator.ts +157 -0
  79. package/src/params.ts +128 -0
  80. package/src/playwright.ts +236 -0
  81. package/src/replay.ts +153 -0
  82. package/src/route.ts +26 -0
  83. package/src/session.ts +178 -0
  84. package/src/snapshot.ts +46 -0
package/src/agent.ts ADDED
@@ -0,0 +1,525 @@
1
+ import { type InferOutput, type StandardSchemaV1, validateStandard } from "@nifrajs/core/schema"
2
+ import { type Cache, type CacheEntry, performedOf, type RecordedAction } from "./cache.ts"
3
+ import {
4
+ type ActDecision,
5
+ type PerformedDecision,
6
+ parseActDecision,
7
+ parseExtraction,
8
+ parseVerdict,
9
+ targetSelector,
10
+ } from "./decision.ts"
11
+ import { type Clock, E2EError, type Engine } from "./engine.ts"
12
+ import { diffEvidence } from "./evidence.ts"
13
+ import {
14
+ forModel,
15
+ forPage,
16
+ type NamedSecret,
17
+ type Params,
18
+ redactSecrets,
19
+ Secret,
20
+ templatize,
21
+ } from "./params.ts"
22
+ import { replayEntry, type StopReason, visibleCount, withoutNth } from "./replay.ts"
23
+ import { normalizeRoute } from "./route.ts"
24
+ import { pruneSnapshot } from "./snapshot.ts"
25
+
26
+ /** One thing the agent tried inside a goal, and whether the page took it. */
27
+ export interface ActionRecord {
28
+ readonly action: PerformedDecision
29
+ readonly ok: boolean
30
+ readonly error?: string
31
+ }
32
+
33
+ /** A recorded run that stopped partway: its actions are already done, so do not redo them. */
34
+ export interface HandOff {
35
+ readonly reason: StopReason
36
+ readonly detail: string
37
+ /** How many actions at the start of `history` the replay performed. */
38
+ readonly performed: number
39
+ }
40
+
41
+ export interface ActRequest {
42
+ readonly kind: "act"
43
+ readonly goal: string
44
+ readonly context?: string
45
+ readonly snapshot: string
46
+ readonly history: readonly ActionRecord[]
47
+ /** Names of params whose values stay out of the goal and the snapshot; write `{name}` to type one. */
48
+ readonly secrets: readonly string[]
49
+ readonly modelCallsLeft: number
50
+ readonly actionsLeft: number
51
+ readonly handoff: HandOff | undefined
52
+ readonly signal: AbortSignal
53
+ }
54
+
55
+ export interface JudgeRequest {
56
+ readonly kind: "assert" | "waitFor"
57
+ readonly statement: string
58
+ readonly context?: string
59
+ readonly snapshot: string
60
+ readonly secrets: readonly string[]
61
+ readonly signal: AbortSignal
62
+ }
63
+
64
+ export interface ExtractRequest {
65
+ readonly kind: "extract"
66
+ readonly prompt: string
67
+ readonly context?: string
68
+ readonly snapshot: string
69
+ readonly secrets: readonly string[]
70
+ /** The JSON Schema the caller supplied, to put in the prompt; the answer is validated against the real schema. */
71
+ readonly jsonSchema?: unknown
72
+ readonly signal: AbortSignal
73
+ }
74
+
75
+ export type ModelRequest = ActRequest | JudgeRequest | ExtractRequest
76
+
77
+ /**
78
+ * The seam to a model. Answer an `act` request with a decision (`{ kind: "tap", target }`, `fill`,
79
+ * `press`, `goto`, `done` or `fail`), an `assert` or `waitFor` with `{ pass, reason }`, and an
80
+ * `extract` with `{ value }`. The answer is unknown until the runner has parsed it. Nothing here
81
+ * names a vendor; the caller owns credentials, retries and cost.
82
+ */
83
+ export interface E2EModel {
84
+ decide(request: ModelRequest): unknown | PromiseLike<unknown>
85
+ }
86
+
87
+ export interface StepOptions {
88
+ readonly params?: Params
89
+ readonly signal?: AbortSignal
90
+ }
91
+
92
+ export interface ActOptions extends StepOptions {
93
+ /** Model decisions one goal may use. Default 12. */
94
+ readonly maxModelCalls?: number
95
+ /** Page actions one goal may take, failed ones included. Default 25. */
96
+ readonly maxActions?: number
97
+ /** ms for the whole goal on the session clock. Default 120000. */
98
+ readonly timeout?: number
99
+ }
100
+
101
+ export interface WaitForOptions extends StepOptions {
102
+ /** ms to wait for the statement to hold. Default 30000. */
103
+ readonly timeout?: number
104
+ }
105
+
106
+ export interface ExtractOptions<S extends StandardSchemaV1> extends StepOptions {
107
+ readonly schema: S
108
+ readonly jsonSchema?: unknown
109
+ }
110
+
111
+ export interface ActResult {
112
+ /** `replay`: no model call. `handoff`: a recording stopped and the model finished the goal. */
113
+ readonly source: "model" | "replay" | "handoff"
114
+ readonly modelCalls: number
115
+ /** The actions the page accepted, in order. */
116
+ readonly actions: readonly PerformedDecision[]
117
+ readonly summary?: string
118
+ }
119
+
120
+ /** What the replay cache did for this session, and how many model calls it still took. */
121
+ export interface AgentReport {
122
+ readonly replayed: number
123
+ readonly handedOff: number
124
+ readonly missed: number
125
+ readonly modelCalls: number
126
+ }
127
+
128
+ export function formatReport(report: AgentReport): string {
129
+ return `Cache: ${report.replayed} replayed - ${report.handedOff} handed off - ${report.missed} missed (${report.modelCalls} model calls)`
130
+ }
131
+
132
+ export interface Agent {
133
+ report(): AgentReport
134
+ act(goal: string, options?: ActOptions): Promise<ActResult>
135
+ assert(statement: string, options?: StepOptions): Promise<void>
136
+ waitFor(statement: string, options?: WaitForOptions): Promise<void>
137
+ extract<S extends StandardSchemaV1>(
138
+ prompt: string,
139
+ options: ExtractOptions<S>,
140
+ ): Promise<InferOutput<S>>
141
+ }
142
+
143
+ export interface AgentDeps {
144
+ readonly engine: Engine
145
+ readonly clock: Clock
146
+ readonly pollInterval: number
147
+ readonly actionTimeout: number
148
+ readonly model: E2EModel | undefined
149
+ readonly context: string | undefined
150
+ readonly snapshotChars: number
151
+ /** Origin of the app under test. */
152
+ readonly origin: string
153
+ readonly cache: Cache | undefined
154
+ /** Opens a same-origin path; refuses any other. */
155
+ readonly navigate: (path: string) => Promise<void>
156
+ }
157
+
158
+ const NEVER = new AbortController().signal
159
+
160
+ function withContext(context: string | undefined): { context?: string } {
161
+ return context === undefined ? {} : { context }
162
+ }
163
+
164
+ function entryOf(
165
+ goal: string,
166
+ startRoute: string,
167
+ actions: readonly RecordedAction[],
168
+ before: string,
169
+ after: string,
170
+ route: string,
171
+ params: Params,
172
+ ): CacheEntry {
173
+ const changes = diffEvidence(before, after)
174
+ return {
175
+ version: 1,
176
+ instruction: goal,
177
+ startRoute,
178
+ actions,
179
+ end: {
180
+ route,
181
+ appeared: changes.appeared.map((line) => templatize(line, params)),
182
+ disappeared: changes.disappeared.map((line) => templatize(line, params)),
183
+ },
184
+ }
185
+ }
186
+
187
+ export function createAgent(deps: AgentDeps): Agent {
188
+ const { engine, clock, cache } = deps
189
+ const stats = { replayed: 0, handedOff: 0, missed: 0, modelCalls: 0 }
190
+
191
+ const requireModel = (): E2EModel => {
192
+ if (deps.model === undefined) {
193
+ throw new E2EError("MODEL_MISSING", "this step needs a model: pass `model` to e2e()")
194
+ }
195
+ return deps.model
196
+ }
197
+
198
+ const ask = async (model: E2EModel, request: ModelRequest): Promise<unknown> => {
199
+ stats.modelCalls += 1
200
+ return model.decide(request)
201
+ }
202
+
203
+ // Every secret any step has been given stays redacted for the whole session: a value typed in one
204
+ // step is still on the page when a later step, which was not handed it, takes a snapshot. They are
205
+ // kept by value, so two steps that reuse a name for different secrets are both redacted.
206
+ const vault: NamedSecret[] = []
207
+ const remember = (params: Params): void => {
208
+ for (const [name, param] of Object.entries(params)) {
209
+ if (!(param instanceof Secret)) continue
210
+ const known = vault.some(([, held]) => Secret.reveal(held) === Secret.reveal(param))
211
+ if (!known) vault.push([name, param])
212
+ }
213
+ }
214
+ const scrub = (text: string): string => redactSecrets(text, vault)
215
+
216
+ const observe = async (): Promise<{ raw: string; shown: string }> => {
217
+ const raw = scrub(await engine.snapshot())
218
+ return { raw, shown: pruneSnapshot(raw, deps.snapshotChars) }
219
+ }
220
+ const look = async (): Promise<string> => (await observe()).shown
221
+
222
+ interface Taken {
223
+ readonly record: ActionRecord
224
+ readonly recorded: RecordedAction | undefined
225
+ }
226
+
227
+ const perform = async (
228
+ decision: PerformedDecision,
229
+ params: Params,
230
+ recording: boolean,
231
+ ): Promise<Taken> => {
232
+ try {
233
+ if (decision.kind === "goto") {
234
+ await deps.navigate(decision.path)
235
+ return {
236
+ record: { action: decision, ok: true },
237
+ recorded: recording
238
+ ? { kind: "goto", path: templatize(decision.path, params) }
239
+ : undefined,
240
+ }
241
+ }
242
+ const selector = targetSelector(decision.target)
243
+ const matches = recording ? await visibleCount(engine, withoutNth(selector)) : 1
244
+ let recorded: RecordedAction
245
+ if (decision.kind === "tap") {
246
+ await engine.act(selector, { kind: "tap" }, deps.actionTimeout)
247
+ recorded = { kind: "tap", target: decision.target, matches }
248
+ } else if (decision.kind === "fill") {
249
+ const value = forPage(decision.value, params)
250
+ await engine.act(selector, { kind: "fill", value }, deps.actionTimeout)
251
+ recorded = {
252
+ kind: "fill",
253
+ target: decision.target,
254
+ matches,
255
+ value: templatize(decision.value, params),
256
+ }
257
+ } else {
258
+ await engine.act(selector, { kind: "press", key: decision.key }, deps.actionTimeout)
259
+ recorded = { kind: "press", target: decision.target, matches, key: decision.key }
260
+ }
261
+ return { record: { action: decision, ok: true }, recorded: recording ? recorded : undefined }
262
+ } catch (error) {
263
+ if (error instanceof E2EError || error instanceof TypeError) {
264
+ return {
265
+ record: { action: decision, ok: false, error: scrub(error.message) },
266
+ recorded: undefined,
267
+ }
268
+ }
269
+ throw error
270
+ }
271
+ }
272
+
273
+ return {
274
+ report: () => ({ ...stats }),
275
+
276
+ async act(goal, options = {}) {
277
+ const params = options.params ?? {}
278
+ remember(params)
279
+ const { text, secrets } = forModel(goal, params)
280
+ const maxCalls = options.maxModelCalls ?? 12
281
+ const maxActions = options.maxActions ?? 25
282
+ const deadline = clock.now() + (options.timeout ?? 120_000)
283
+ const recording = cache?.canWrite === true
284
+ const key =
285
+ cache === undefined || cache.mode === "off"
286
+ ? undefined
287
+ : cache.key(goal, params, deps.context)
288
+ const startRoute = normalizeRoute(await engine.url(), deps.origin)
289
+ const before = recording ? (await observe()).raw : ""
290
+
291
+ const history: ActionRecord[] = []
292
+ const taken: RecordedAction[] = []
293
+ let handoff: HandOff | undefined
294
+ if (cache !== undefined && key !== undefined) {
295
+ const entry = await cache.read(key)
296
+ if (entry === undefined) {
297
+ if (cache.mode === "strict") {
298
+ throw new E2EError(
299
+ "REPLAY_MISSING",
300
+ `goal "${text}" has no recording and the cache is strict`,
301
+ )
302
+ }
303
+ stats.missed += 1
304
+ } else {
305
+ const outcome = await replayEntry(
306
+ { ...deps, observe: async () => (await observe()).raw },
307
+ entry,
308
+ params,
309
+ )
310
+ if (outcome.ok) {
311
+ stats.replayed += 1
312
+ cache.replayed(key)
313
+ return { source: "replay", modelCalls: 0, actions: entry.actions.map(performedOf) }
314
+ }
315
+ if (cache.mode === "strict") {
316
+ throw new E2EError(
317
+ "REPLAY_STALE",
318
+ `goal "${text}" no longer replays (${outcome.reason}): ${outcome.detail}`,
319
+ )
320
+ }
321
+ stats.handedOff += 1
322
+ handoff = {
323
+ reason: outcome.reason,
324
+ detail: scrub(outcome.detail),
325
+ performed: outcome.performed.length,
326
+ }
327
+ for (const action of outcome.performed) {
328
+ taken.push(action)
329
+ history.push({ action: performedOf(action), ok: true })
330
+ }
331
+ }
332
+ }
333
+
334
+ const runGoal = async (): Promise<ActResult> => {
335
+ let calls = 0
336
+ for (;;) {
337
+ if (clock.now() >= deadline) {
338
+ throw new E2EError("STEP_TIMEOUT", `goal "${text}" ran past its time limit`)
339
+ }
340
+ if (calls >= maxCalls) {
341
+ throw new E2EError(
342
+ "STEP_BUDGET_EXCEEDED",
343
+ `goal "${text}" used its ${maxCalls} model calls`,
344
+ )
345
+ }
346
+ const model = requireModel()
347
+ const seen = await observe()
348
+ calls += 1
349
+ const raw = await ask(model, {
350
+ kind: "act",
351
+ goal: text,
352
+ ...withContext(deps.context),
353
+ snapshot: seen.shown,
354
+ history,
355
+ secrets,
356
+ modelCallsLeft: maxCalls - calls,
357
+ actionsLeft: maxActions - history.length,
358
+ handoff,
359
+ signal: options.signal ?? NEVER,
360
+ })
361
+ const parsed = parseActDecision(raw)
362
+ if (!parsed.ok) {
363
+ throw new E2EError(
364
+ "MODEL_INVALID",
365
+ `the model's decision was refused: ${parsed.reason}`,
366
+ )
367
+ }
368
+ const decision: ActDecision = parsed.value
369
+ if (decision.kind === "fail") {
370
+ throw new E2EError("STEP_FAILED", `goal "${text}" failed: ${decision.reason}`)
371
+ }
372
+ if (decision.kind === "done") {
373
+ if (recording && cache !== undefined && key !== undefined && taken.length > 0) {
374
+ cache.stage(
375
+ key,
376
+ entryOf(
377
+ goal,
378
+ startRoute,
379
+ taken,
380
+ before,
381
+ seen.raw,
382
+ normalizeRoute(await engine.url(), deps.origin),
383
+ params,
384
+ ),
385
+ )
386
+ }
387
+ const result: {
388
+ source: ActResult["source"]
389
+ modelCalls: number
390
+ actions: PerformedDecision[]
391
+ summary?: string
392
+ } = {
393
+ source: handoff === undefined ? "model" : "handoff",
394
+ modelCalls: calls,
395
+ actions: history.filter((entry) => entry.ok).map((entry) => entry.action),
396
+ }
397
+ if (decision.summary !== undefined) result.summary = decision.summary
398
+ return result
399
+ }
400
+ if (history.length >= maxActions) {
401
+ throw new E2EError(
402
+ "STEP_BUDGET_EXCEEDED",
403
+ `goal "${text}" used its ${maxActions} actions`,
404
+ )
405
+ }
406
+ const step = await perform(decision, params, recording)
407
+ history.push(step.record)
408
+ if (step.recorded !== undefined) taken.push(step.recorded)
409
+ }
410
+ }
411
+
412
+ try {
413
+ return await runGoal()
414
+ } catch (error) {
415
+ const missingModel = error instanceof E2EError && error.code === "MODEL_MISSING"
416
+ if (handoff !== undefined && !missingModel && cache !== undefined && key !== undefined) {
417
+ await cache.discard(key)
418
+ }
419
+ throw error
420
+ }
421
+ },
422
+
423
+ async assert(statement, options = {}) {
424
+ const model = requireModel()
425
+ const params = options.params ?? {}
426
+ remember(params)
427
+ const { text, secrets } = forModel(statement, params)
428
+ const raw = await ask(model, {
429
+ kind: "assert",
430
+ statement: text,
431
+ ...withContext(deps.context),
432
+ snapshot: await look(),
433
+ secrets,
434
+ signal: options.signal ?? NEVER,
435
+ })
436
+ const verdict = parseVerdict(raw)
437
+ if (!verdict.ok) {
438
+ throw new E2EError("MODEL_INVALID", `the model's verdict was refused: ${verdict.reason}`)
439
+ }
440
+ if (verdict.value.pass === true) {
441
+ await cache?.onCheck(true)
442
+ return
443
+ }
444
+ const reason = verdict.value.reason === "" ? "" : `: ${verdict.value.reason}`
445
+ if (verdict.value.pass === "inconclusive") {
446
+ throw new E2EError("ASSERTION_INCONCLUSIVE", `could not tell whether "${text}"${reason}`)
447
+ }
448
+ await cache?.onCheck(false)
449
+ throw new E2EError("ASSERTION_FAILED", `"${text}" does not hold${reason}`)
450
+ },
451
+
452
+ async waitFor(statement, options = {}) {
453
+ const model = requireModel()
454
+ const params = options.params ?? {}
455
+ remember(params)
456
+ const { text, secrets } = forModel(statement, params)
457
+ const timeout = options.timeout ?? 30_000
458
+ const deadline = clock.now() + timeout
459
+ let seen: string | undefined
460
+ let lastReason = ""
461
+ for (;;) {
462
+ const snapshot = await look()
463
+ if (snapshot !== seen) {
464
+ seen = snapshot
465
+ const raw = await ask(model, {
466
+ kind: "waitFor",
467
+ statement: text,
468
+ ...withContext(deps.context),
469
+ snapshot,
470
+ secrets,
471
+ signal: options.signal ?? NEVER,
472
+ })
473
+ const verdict = parseVerdict(raw)
474
+ if (!verdict.ok) {
475
+ throw new E2EError(
476
+ "MODEL_INVALID",
477
+ `the model's verdict was refused: ${verdict.reason}`,
478
+ )
479
+ }
480
+ if (verdict.value.pass === true) {
481
+ await cache?.onCheck(true)
482
+ return
483
+ }
484
+ lastReason = verdict.value.reason
485
+ }
486
+ const remaining = deadline - clock.now()
487
+ if (remaining <= 0) {
488
+ await cache?.onCheck(false)
489
+ const reason = lastReason === "" ? "" : `: ${lastReason}`
490
+ throw new E2EError("WAIT_TIMEOUT", `"${text}" did not hold within ${timeout} ms${reason}`)
491
+ }
492
+ await clock.sleep(Math.min(deps.pollInterval, remaining))
493
+ }
494
+ },
495
+
496
+ async extract(prompt, options) {
497
+ const model = requireModel()
498
+ const params = options.params ?? {}
499
+ remember(params)
500
+ const { text, secrets } = forModel(prompt, params)
501
+ const raw = await ask(model, {
502
+ kind: "extract",
503
+ prompt: text,
504
+ ...withContext(deps.context),
505
+ snapshot: await look(),
506
+ secrets,
507
+ ...(options.jsonSchema === undefined ? {} : { jsonSchema: options.jsonSchema }),
508
+ signal: options.signal ?? NEVER,
509
+ })
510
+ const extraction = parseExtraction(raw)
511
+ if (!extraction.ok) {
512
+ throw new E2EError("MODEL_INVALID", `the model's answer was refused: ${extraction.reason}`)
513
+ }
514
+ const checked = await validateStandard(options.schema, extraction.value)
515
+ if (!checked.ok) {
516
+ const issues = checked.issues.map((issue) => issue.message).join("; ")
517
+ throw new E2EError(
518
+ "EXTRACT_INVALID",
519
+ `the extracted data does not match the schema: ${issues}`,
520
+ )
521
+ }
522
+ return checked.value
523
+ },
524
+ }
525
+ }