@nifrajs/e2e 0.0.0-placeholder → 5.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/LICENSE +21 -0
- package/README.md +128 -0
- package/dist/agent.d.ts +119 -0
- package/dist/agent.d.ts.map +1 -0
- package/dist/agent.js +315 -0
- package/dist/agent.js.map +1 -0
- package/dist/cache.d.ts +84 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +166 -0
- package/dist/cache.js.map +1 -0
- package/dist/decision.d.ts +54 -0
- package/dist/decision.d.ts.map +1 -0
- package/dist/decision.js +122 -0
- package/dist/decision.js.map +1 -0
- package/dist/engine.d.ts +75 -0
- package/dist/engine.d.ts.map +1 -0
- package/dist/engine.js +13 -0
- package/dist/engine.js.map +1 -0
- package/dist/evidence.d.ts +14 -0
- package/dist/evidence.d.ts.map +1 -0
- package/dist/evidence.js +52 -0
- package/dist/evidence.js.map +1 -0
- package/dist/expect.d.ts +28 -0
- package/dist/expect.d.ts.map +1 -0
- package/dist/expect.js +88 -0
- package/dist/expect.js.map +1 -0
- package/dist/fake.d.ts +35 -0
- package/dist/fake.d.ts.map +1 -0
- package/dist/fake.js +129 -0
- package/dist/fake.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/record.d.ts +3 -0
- package/dist/internal/record.d.ts.map +1 -0
- package/dist/internal/record.js +6 -0
- package/dist/internal/record.js.map +1 -0
- package/dist/locator.d.ts +53 -0
- package/dist/locator.d.ts.map +1 -0
- package/dist/locator.js +103 -0
- package/dist/locator.js.map +1 -0
- package/dist/params.d.ts +44 -0
- package/dist/params.d.ts.map +1 -0
- package/dist/params.js +108 -0
- package/dist/params.js.map +1 -0
- package/dist/playwright.d.ts +72 -0
- package/dist/playwright.d.ts.map +1 -0
- package/dist/playwright.js +169 -0
- package/dist/playwright.js.map +1 -0
- package/dist/replay.d.ts +34 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/replay.js +111 -0
- package/dist/replay.js.map +1 -0
- package/dist/route.d.ts +7 -0
- package/dist/route.d.ts.map +1 -0
- package/dist/route.js +26 -0
- package/dist/route.js.map +1 -0
- package/dist/session.d.ts +62 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +105 -0
- package/dist/session.js.map +1 -0
- package/dist/snapshot.d.ts +8 -0
- package/dist/snapshot.d.ts.map +1 -0
- package/dist/snapshot.js +49 -0
- package/dist/snapshot.js.map +1 -0
- package/package.json +75 -1
- package/src/agent.ts +525 -0
- package/src/cache.ts +229 -0
- package/src/decision.ts +152 -0
- package/src/engine.ts +92 -0
- package/src/evidence.ts +58 -0
- package/src/expect.ts +171 -0
- package/src/fake.ts +185 -0
- package/src/index.ts +64 -0
- package/src/internal/record.ts +5 -0
- package/src/locator.ts +157 -0
- package/src/params.ts +128 -0
- package/src/playwright.ts +236 -0
- package/src/replay.ts +153 -0
- package/src/route.ts +26 -0
- package/src/session.ts +178 -0
- package/src/snapshot.ts +46 -0
package/src/agent.ts
ADDED
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
import { type InferOutput, type StandardSchemaV1, validateStandard } from "@nifrajs/core/schema"
|
|
2
|
+
import { type Cache, type CacheEntry, performedOf, type RecordedAction } from "./cache.ts"
|
|
3
|
+
import {
|
|
4
|
+
type ActDecision,
|
|
5
|
+
type PerformedDecision,
|
|
6
|
+
parseActDecision,
|
|
7
|
+
parseExtraction,
|
|
8
|
+
parseVerdict,
|
|
9
|
+
targetSelector,
|
|
10
|
+
} from "./decision.ts"
|
|
11
|
+
import { type Clock, E2EError, type Engine } from "./engine.ts"
|
|
12
|
+
import { diffEvidence } from "./evidence.ts"
|
|
13
|
+
import {
|
|
14
|
+
forModel,
|
|
15
|
+
forPage,
|
|
16
|
+
type NamedSecret,
|
|
17
|
+
type Params,
|
|
18
|
+
redactSecrets,
|
|
19
|
+
Secret,
|
|
20
|
+
templatize,
|
|
21
|
+
} from "./params.ts"
|
|
22
|
+
import { replayEntry, type StopReason, visibleCount, withoutNth } from "./replay.ts"
|
|
23
|
+
import { normalizeRoute } from "./route.ts"
|
|
24
|
+
import { pruneSnapshot } from "./snapshot.ts"
|
|
25
|
+
|
|
26
|
+
/** One thing the agent tried inside a goal, and whether the page took it. */
|
|
27
|
+
export interface ActionRecord {
|
|
28
|
+
readonly action: PerformedDecision
|
|
29
|
+
readonly ok: boolean
|
|
30
|
+
readonly error?: string
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A recorded run that stopped partway: its actions are already done, so do not redo them. */
|
|
34
|
+
export interface HandOff {
|
|
35
|
+
readonly reason: StopReason
|
|
36
|
+
readonly detail: string
|
|
37
|
+
/** How many actions at the start of `history` the replay performed. */
|
|
38
|
+
readonly performed: number
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface ActRequest {
|
|
42
|
+
readonly kind: "act"
|
|
43
|
+
readonly goal: string
|
|
44
|
+
readonly context?: string
|
|
45
|
+
readonly snapshot: string
|
|
46
|
+
readonly history: readonly ActionRecord[]
|
|
47
|
+
/** Names of params whose values stay out of the goal and the snapshot; write `{name}` to type one. */
|
|
48
|
+
readonly secrets: readonly string[]
|
|
49
|
+
readonly modelCallsLeft: number
|
|
50
|
+
readonly actionsLeft: number
|
|
51
|
+
readonly handoff: HandOff | undefined
|
|
52
|
+
readonly signal: AbortSignal
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface JudgeRequest {
|
|
56
|
+
readonly kind: "assert" | "waitFor"
|
|
57
|
+
readonly statement: string
|
|
58
|
+
readonly context?: string
|
|
59
|
+
readonly snapshot: string
|
|
60
|
+
readonly secrets: readonly string[]
|
|
61
|
+
readonly signal: AbortSignal
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface ExtractRequest {
|
|
65
|
+
readonly kind: "extract"
|
|
66
|
+
readonly prompt: string
|
|
67
|
+
readonly context?: string
|
|
68
|
+
readonly snapshot: string
|
|
69
|
+
readonly secrets: readonly string[]
|
|
70
|
+
/** The JSON Schema the caller supplied, to put in the prompt; the answer is validated against the real schema. */
|
|
71
|
+
readonly jsonSchema?: unknown
|
|
72
|
+
readonly signal: AbortSignal
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export type ModelRequest = ActRequest | JudgeRequest | ExtractRequest
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The seam to a model. Answer an `act` request with a decision (`{ kind: "tap", target }`, `fill`,
|
|
79
|
+
* `press`, `goto`, `done` or `fail`), an `assert` or `waitFor` with `{ pass, reason }`, and an
|
|
80
|
+
* `extract` with `{ value }`. The answer is unknown until the runner has parsed it. Nothing here
|
|
81
|
+
* names a vendor; the caller owns credentials, retries and cost.
|
|
82
|
+
*/
|
|
83
|
+
export interface E2EModel {
|
|
84
|
+
decide(request: ModelRequest): unknown | PromiseLike<unknown>
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export interface StepOptions {
|
|
88
|
+
readonly params?: Params
|
|
89
|
+
readonly signal?: AbortSignal
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export interface ActOptions extends StepOptions {
|
|
93
|
+
/** Model decisions one goal may use. Default 12. */
|
|
94
|
+
readonly maxModelCalls?: number
|
|
95
|
+
/** Page actions one goal may take, failed ones included. Default 25. */
|
|
96
|
+
readonly maxActions?: number
|
|
97
|
+
/** ms for the whole goal on the session clock. Default 120000. */
|
|
98
|
+
readonly timeout?: number
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface WaitForOptions extends StepOptions {
|
|
102
|
+
/** ms to wait for the statement to hold. Default 30000. */
|
|
103
|
+
readonly timeout?: number
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface ExtractOptions<S extends StandardSchemaV1> extends StepOptions {
|
|
107
|
+
readonly schema: S
|
|
108
|
+
readonly jsonSchema?: unknown
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export interface ActResult {
|
|
112
|
+
/** `replay`: no model call. `handoff`: a recording stopped and the model finished the goal. */
|
|
113
|
+
readonly source: "model" | "replay" | "handoff"
|
|
114
|
+
readonly modelCalls: number
|
|
115
|
+
/** The actions the page accepted, in order. */
|
|
116
|
+
readonly actions: readonly PerformedDecision[]
|
|
117
|
+
readonly summary?: string
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** What the replay cache did for this session, and how many model calls it still took. */
|
|
121
|
+
export interface AgentReport {
|
|
122
|
+
readonly replayed: number
|
|
123
|
+
readonly handedOff: number
|
|
124
|
+
readonly missed: number
|
|
125
|
+
readonly modelCalls: number
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function formatReport(report: AgentReport): string {
|
|
129
|
+
return `Cache: ${report.replayed} replayed - ${report.handedOff} handed off - ${report.missed} missed (${report.modelCalls} model calls)`
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export interface Agent {
|
|
133
|
+
report(): AgentReport
|
|
134
|
+
act(goal: string, options?: ActOptions): Promise<ActResult>
|
|
135
|
+
assert(statement: string, options?: StepOptions): Promise<void>
|
|
136
|
+
waitFor(statement: string, options?: WaitForOptions): Promise<void>
|
|
137
|
+
extract<S extends StandardSchemaV1>(
|
|
138
|
+
prompt: string,
|
|
139
|
+
options: ExtractOptions<S>,
|
|
140
|
+
): Promise<InferOutput<S>>
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export interface AgentDeps {
|
|
144
|
+
readonly engine: Engine
|
|
145
|
+
readonly clock: Clock
|
|
146
|
+
readonly pollInterval: number
|
|
147
|
+
readonly actionTimeout: number
|
|
148
|
+
readonly model: E2EModel | undefined
|
|
149
|
+
readonly context: string | undefined
|
|
150
|
+
readonly snapshotChars: number
|
|
151
|
+
/** Origin of the app under test. */
|
|
152
|
+
readonly origin: string
|
|
153
|
+
readonly cache: Cache | undefined
|
|
154
|
+
/** Opens a same-origin path; refuses any other. */
|
|
155
|
+
readonly navigate: (path: string) => Promise<void>
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const NEVER = new AbortController().signal
|
|
159
|
+
|
|
160
|
+
function withContext(context: string | undefined): { context?: string } {
|
|
161
|
+
return context === undefined ? {} : { context }
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function entryOf(
|
|
165
|
+
goal: string,
|
|
166
|
+
startRoute: string,
|
|
167
|
+
actions: readonly RecordedAction[],
|
|
168
|
+
before: string,
|
|
169
|
+
after: string,
|
|
170
|
+
route: string,
|
|
171
|
+
params: Params,
|
|
172
|
+
): CacheEntry {
|
|
173
|
+
const changes = diffEvidence(before, after)
|
|
174
|
+
return {
|
|
175
|
+
version: 1,
|
|
176
|
+
instruction: goal,
|
|
177
|
+
startRoute,
|
|
178
|
+
actions,
|
|
179
|
+
end: {
|
|
180
|
+
route,
|
|
181
|
+
appeared: changes.appeared.map((line) => templatize(line, params)),
|
|
182
|
+
disappeared: changes.disappeared.map((line) => templatize(line, params)),
|
|
183
|
+
},
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function createAgent(deps: AgentDeps): Agent {
|
|
188
|
+
const { engine, clock, cache } = deps
|
|
189
|
+
const stats = { replayed: 0, handedOff: 0, missed: 0, modelCalls: 0 }
|
|
190
|
+
|
|
191
|
+
const requireModel = (): E2EModel => {
|
|
192
|
+
if (deps.model === undefined) {
|
|
193
|
+
throw new E2EError("MODEL_MISSING", "this step needs a model: pass `model` to e2e()")
|
|
194
|
+
}
|
|
195
|
+
return deps.model
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const ask = async (model: E2EModel, request: ModelRequest): Promise<unknown> => {
|
|
199
|
+
stats.modelCalls += 1
|
|
200
|
+
return model.decide(request)
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Every secret any step has been given stays redacted for the whole session: a value typed in one
|
|
204
|
+
// step is still on the page when a later step, which was not handed it, takes a snapshot. They are
|
|
205
|
+
// kept by value, so two steps that reuse a name for different secrets are both redacted.
|
|
206
|
+
const vault: NamedSecret[] = []
|
|
207
|
+
const remember = (params: Params): void => {
|
|
208
|
+
for (const [name, param] of Object.entries(params)) {
|
|
209
|
+
if (!(param instanceof Secret)) continue
|
|
210
|
+
const known = vault.some(([, held]) => Secret.reveal(held) === Secret.reveal(param))
|
|
211
|
+
if (!known) vault.push([name, param])
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
const scrub = (text: string): string => redactSecrets(text, vault)
|
|
215
|
+
|
|
216
|
+
const observe = async (): Promise<{ raw: string; shown: string }> => {
|
|
217
|
+
const raw = scrub(await engine.snapshot())
|
|
218
|
+
return { raw, shown: pruneSnapshot(raw, deps.snapshotChars) }
|
|
219
|
+
}
|
|
220
|
+
const look = async (): Promise<string> => (await observe()).shown
|
|
221
|
+
|
|
222
|
+
interface Taken {
|
|
223
|
+
readonly record: ActionRecord
|
|
224
|
+
readonly recorded: RecordedAction | undefined
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const perform = async (
|
|
228
|
+
decision: PerformedDecision,
|
|
229
|
+
params: Params,
|
|
230
|
+
recording: boolean,
|
|
231
|
+
): Promise<Taken> => {
|
|
232
|
+
try {
|
|
233
|
+
if (decision.kind === "goto") {
|
|
234
|
+
await deps.navigate(decision.path)
|
|
235
|
+
return {
|
|
236
|
+
record: { action: decision, ok: true },
|
|
237
|
+
recorded: recording
|
|
238
|
+
? { kind: "goto", path: templatize(decision.path, params) }
|
|
239
|
+
: undefined,
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
const selector = targetSelector(decision.target)
|
|
243
|
+
const matches = recording ? await visibleCount(engine, withoutNth(selector)) : 1
|
|
244
|
+
let recorded: RecordedAction
|
|
245
|
+
if (decision.kind === "tap") {
|
|
246
|
+
await engine.act(selector, { kind: "tap" }, deps.actionTimeout)
|
|
247
|
+
recorded = { kind: "tap", target: decision.target, matches }
|
|
248
|
+
} else if (decision.kind === "fill") {
|
|
249
|
+
const value = forPage(decision.value, params)
|
|
250
|
+
await engine.act(selector, { kind: "fill", value }, deps.actionTimeout)
|
|
251
|
+
recorded = {
|
|
252
|
+
kind: "fill",
|
|
253
|
+
target: decision.target,
|
|
254
|
+
matches,
|
|
255
|
+
value: templatize(decision.value, params),
|
|
256
|
+
}
|
|
257
|
+
} else {
|
|
258
|
+
await engine.act(selector, { kind: "press", key: decision.key }, deps.actionTimeout)
|
|
259
|
+
recorded = { kind: "press", target: decision.target, matches, key: decision.key }
|
|
260
|
+
}
|
|
261
|
+
return { record: { action: decision, ok: true }, recorded: recording ? recorded : undefined }
|
|
262
|
+
} catch (error) {
|
|
263
|
+
if (error instanceof E2EError || error instanceof TypeError) {
|
|
264
|
+
return {
|
|
265
|
+
record: { action: decision, ok: false, error: scrub(error.message) },
|
|
266
|
+
recorded: undefined,
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
throw error
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return {
|
|
274
|
+
report: () => ({ ...stats }),
|
|
275
|
+
|
|
276
|
+
async act(goal, options = {}) {
|
|
277
|
+
const params = options.params ?? {}
|
|
278
|
+
remember(params)
|
|
279
|
+
const { text, secrets } = forModel(goal, params)
|
|
280
|
+
const maxCalls = options.maxModelCalls ?? 12
|
|
281
|
+
const maxActions = options.maxActions ?? 25
|
|
282
|
+
const deadline = clock.now() + (options.timeout ?? 120_000)
|
|
283
|
+
const recording = cache?.canWrite === true
|
|
284
|
+
const key =
|
|
285
|
+
cache === undefined || cache.mode === "off"
|
|
286
|
+
? undefined
|
|
287
|
+
: cache.key(goal, params, deps.context)
|
|
288
|
+
const startRoute = normalizeRoute(await engine.url(), deps.origin)
|
|
289
|
+
const before = recording ? (await observe()).raw : ""
|
|
290
|
+
|
|
291
|
+
const history: ActionRecord[] = []
|
|
292
|
+
const taken: RecordedAction[] = []
|
|
293
|
+
let handoff: HandOff | undefined
|
|
294
|
+
if (cache !== undefined && key !== undefined) {
|
|
295
|
+
const entry = await cache.read(key)
|
|
296
|
+
if (entry === undefined) {
|
|
297
|
+
if (cache.mode === "strict") {
|
|
298
|
+
throw new E2EError(
|
|
299
|
+
"REPLAY_MISSING",
|
|
300
|
+
`goal "${text}" has no recording and the cache is strict`,
|
|
301
|
+
)
|
|
302
|
+
}
|
|
303
|
+
stats.missed += 1
|
|
304
|
+
} else {
|
|
305
|
+
const outcome = await replayEntry(
|
|
306
|
+
{ ...deps, observe: async () => (await observe()).raw },
|
|
307
|
+
entry,
|
|
308
|
+
params,
|
|
309
|
+
)
|
|
310
|
+
if (outcome.ok) {
|
|
311
|
+
stats.replayed += 1
|
|
312
|
+
cache.replayed(key)
|
|
313
|
+
return { source: "replay", modelCalls: 0, actions: entry.actions.map(performedOf) }
|
|
314
|
+
}
|
|
315
|
+
if (cache.mode === "strict") {
|
|
316
|
+
throw new E2EError(
|
|
317
|
+
"REPLAY_STALE",
|
|
318
|
+
`goal "${text}" no longer replays (${outcome.reason}): ${outcome.detail}`,
|
|
319
|
+
)
|
|
320
|
+
}
|
|
321
|
+
stats.handedOff += 1
|
|
322
|
+
handoff = {
|
|
323
|
+
reason: outcome.reason,
|
|
324
|
+
detail: scrub(outcome.detail),
|
|
325
|
+
performed: outcome.performed.length,
|
|
326
|
+
}
|
|
327
|
+
for (const action of outcome.performed) {
|
|
328
|
+
taken.push(action)
|
|
329
|
+
history.push({ action: performedOf(action), ok: true })
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const runGoal = async (): Promise<ActResult> => {
|
|
335
|
+
let calls = 0
|
|
336
|
+
for (;;) {
|
|
337
|
+
if (clock.now() >= deadline) {
|
|
338
|
+
throw new E2EError("STEP_TIMEOUT", `goal "${text}" ran past its time limit`)
|
|
339
|
+
}
|
|
340
|
+
if (calls >= maxCalls) {
|
|
341
|
+
throw new E2EError(
|
|
342
|
+
"STEP_BUDGET_EXCEEDED",
|
|
343
|
+
`goal "${text}" used its ${maxCalls} model calls`,
|
|
344
|
+
)
|
|
345
|
+
}
|
|
346
|
+
const model = requireModel()
|
|
347
|
+
const seen = await observe()
|
|
348
|
+
calls += 1
|
|
349
|
+
const raw = await ask(model, {
|
|
350
|
+
kind: "act",
|
|
351
|
+
goal: text,
|
|
352
|
+
...withContext(deps.context),
|
|
353
|
+
snapshot: seen.shown,
|
|
354
|
+
history,
|
|
355
|
+
secrets,
|
|
356
|
+
modelCallsLeft: maxCalls - calls,
|
|
357
|
+
actionsLeft: maxActions - history.length,
|
|
358
|
+
handoff,
|
|
359
|
+
signal: options.signal ?? NEVER,
|
|
360
|
+
})
|
|
361
|
+
const parsed = parseActDecision(raw)
|
|
362
|
+
if (!parsed.ok) {
|
|
363
|
+
throw new E2EError(
|
|
364
|
+
"MODEL_INVALID",
|
|
365
|
+
`the model's decision was refused: ${parsed.reason}`,
|
|
366
|
+
)
|
|
367
|
+
}
|
|
368
|
+
const decision: ActDecision = parsed.value
|
|
369
|
+
if (decision.kind === "fail") {
|
|
370
|
+
throw new E2EError("STEP_FAILED", `goal "${text}" failed: ${decision.reason}`)
|
|
371
|
+
}
|
|
372
|
+
if (decision.kind === "done") {
|
|
373
|
+
if (recording && cache !== undefined && key !== undefined && taken.length > 0) {
|
|
374
|
+
cache.stage(
|
|
375
|
+
key,
|
|
376
|
+
entryOf(
|
|
377
|
+
goal,
|
|
378
|
+
startRoute,
|
|
379
|
+
taken,
|
|
380
|
+
before,
|
|
381
|
+
seen.raw,
|
|
382
|
+
normalizeRoute(await engine.url(), deps.origin),
|
|
383
|
+
params,
|
|
384
|
+
),
|
|
385
|
+
)
|
|
386
|
+
}
|
|
387
|
+
const result: {
|
|
388
|
+
source: ActResult["source"]
|
|
389
|
+
modelCalls: number
|
|
390
|
+
actions: PerformedDecision[]
|
|
391
|
+
summary?: string
|
|
392
|
+
} = {
|
|
393
|
+
source: handoff === undefined ? "model" : "handoff",
|
|
394
|
+
modelCalls: calls,
|
|
395
|
+
actions: history.filter((entry) => entry.ok).map((entry) => entry.action),
|
|
396
|
+
}
|
|
397
|
+
if (decision.summary !== undefined) result.summary = decision.summary
|
|
398
|
+
return result
|
|
399
|
+
}
|
|
400
|
+
if (history.length >= maxActions) {
|
|
401
|
+
throw new E2EError(
|
|
402
|
+
"STEP_BUDGET_EXCEEDED",
|
|
403
|
+
`goal "${text}" used its ${maxActions} actions`,
|
|
404
|
+
)
|
|
405
|
+
}
|
|
406
|
+
const step = await perform(decision, params, recording)
|
|
407
|
+
history.push(step.record)
|
|
408
|
+
if (step.recorded !== undefined) taken.push(step.recorded)
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
try {
|
|
413
|
+
return await runGoal()
|
|
414
|
+
} catch (error) {
|
|
415
|
+
const missingModel = error instanceof E2EError && error.code === "MODEL_MISSING"
|
|
416
|
+
if (handoff !== undefined && !missingModel && cache !== undefined && key !== undefined) {
|
|
417
|
+
await cache.discard(key)
|
|
418
|
+
}
|
|
419
|
+
throw error
|
|
420
|
+
}
|
|
421
|
+
},
|
|
422
|
+
|
|
423
|
+
async assert(statement, options = {}) {
|
|
424
|
+
const model = requireModel()
|
|
425
|
+
const params = options.params ?? {}
|
|
426
|
+
remember(params)
|
|
427
|
+
const { text, secrets } = forModel(statement, params)
|
|
428
|
+
const raw = await ask(model, {
|
|
429
|
+
kind: "assert",
|
|
430
|
+
statement: text,
|
|
431
|
+
...withContext(deps.context),
|
|
432
|
+
snapshot: await look(),
|
|
433
|
+
secrets,
|
|
434
|
+
signal: options.signal ?? NEVER,
|
|
435
|
+
})
|
|
436
|
+
const verdict = parseVerdict(raw)
|
|
437
|
+
if (!verdict.ok) {
|
|
438
|
+
throw new E2EError("MODEL_INVALID", `the model's verdict was refused: ${verdict.reason}`)
|
|
439
|
+
}
|
|
440
|
+
if (verdict.value.pass === true) {
|
|
441
|
+
await cache?.onCheck(true)
|
|
442
|
+
return
|
|
443
|
+
}
|
|
444
|
+
const reason = verdict.value.reason === "" ? "" : `: ${verdict.value.reason}`
|
|
445
|
+
if (verdict.value.pass === "inconclusive") {
|
|
446
|
+
throw new E2EError("ASSERTION_INCONCLUSIVE", `could not tell whether "${text}"${reason}`)
|
|
447
|
+
}
|
|
448
|
+
await cache?.onCheck(false)
|
|
449
|
+
throw new E2EError("ASSERTION_FAILED", `"${text}" does not hold${reason}`)
|
|
450
|
+
},
|
|
451
|
+
|
|
452
|
+
async waitFor(statement, options = {}) {
|
|
453
|
+
const model = requireModel()
|
|
454
|
+
const params = options.params ?? {}
|
|
455
|
+
remember(params)
|
|
456
|
+
const { text, secrets } = forModel(statement, params)
|
|
457
|
+
const timeout = options.timeout ?? 30_000
|
|
458
|
+
const deadline = clock.now() + timeout
|
|
459
|
+
let seen: string | undefined
|
|
460
|
+
let lastReason = ""
|
|
461
|
+
for (;;) {
|
|
462
|
+
const snapshot = await look()
|
|
463
|
+
if (snapshot !== seen) {
|
|
464
|
+
seen = snapshot
|
|
465
|
+
const raw = await ask(model, {
|
|
466
|
+
kind: "waitFor",
|
|
467
|
+
statement: text,
|
|
468
|
+
...withContext(deps.context),
|
|
469
|
+
snapshot,
|
|
470
|
+
secrets,
|
|
471
|
+
signal: options.signal ?? NEVER,
|
|
472
|
+
})
|
|
473
|
+
const verdict = parseVerdict(raw)
|
|
474
|
+
if (!verdict.ok) {
|
|
475
|
+
throw new E2EError(
|
|
476
|
+
"MODEL_INVALID",
|
|
477
|
+
`the model's verdict was refused: ${verdict.reason}`,
|
|
478
|
+
)
|
|
479
|
+
}
|
|
480
|
+
if (verdict.value.pass === true) {
|
|
481
|
+
await cache?.onCheck(true)
|
|
482
|
+
return
|
|
483
|
+
}
|
|
484
|
+
lastReason = verdict.value.reason
|
|
485
|
+
}
|
|
486
|
+
const remaining = deadline - clock.now()
|
|
487
|
+
if (remaining <= 0) {
|
|
488
|
+
await cache?.onCheck(false)
|
|
489
|
+
const reason = lastReason === "" ? "" : `: ${lastReason}`
|
|
490
|
+
throw new E2EError("WAIT_TIMEOUT", `"${text}" did not hold within ${timeout} ms${reason}`)
|
|
491
|
+
}
|
|
492
|
+
await clock.sleep(Math.min(deps.pollInterval, remaining))
|
|
493
|
+
}
|
|
494
|
+
},
|
|
495
|
+
|
|
496
|
+
async extract(prompt, options) {
|
|
497
|
+
const model = requireModel()
|
|
498
|
+
const params = options.params ?? {}
|
|
499
|
+
remember(params)
|
|
500
|
+
const { text, secrets } = forModel(prompt, params)
|
|
501
|
+
const raw = await ask(model, {
|
|
502
|
+
kind: "extract",
|
|
503
|
+
prompt: text,
|
|
504
|
+
...withContext(deps.context),
|
|
505
|
+
snapshot: await look(),
|
|
506
|
+
secrets,
|
|
507
|
+
...(options.jsonSchema === undefined ? {} : { jsonSchema: options.jsonSchema }),
|
|
508
|
+
signal: options.signal ?? NEVER,
|
|
509
|
+
})
|
|
510
|
+
const extraction = parseExtraction(raw)
|
|
511
|
+
if (!extraction.ok) {
|
|
512
|
+
throw new E2EError("MODEL_INVALID", `the model's answer was refused: ${extraction.reason}`)
|
|
513
|
+
}
|
|
514
|
+
const checked = await validateStandard(options.schema, extraction.value)
|
|
515
|
+
if (!checked.ok) {
|
|
516
|
+
const issues = checked.issues.map((issue) => issue.message).join("; ")
|
|
517
|
+
throw new E2EError(
|
|
518
|
+
"EXTRACT_INVALID",
|
|
519
|
+
`the extracted data does not match the schema: ${issues}`,
|
|
520
|
+
)
|
|
521
|
+
}
|
|
522
|
+
return checked.value
|
|
523
|
+
},
|
|
524
|
+
}
|
|
525
|
+
}
|