e2e 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/.build.json +1 -1
- package/dist/agent/act.js +7 -4
- package/dist/agent/action-dispatcher.js +4 -5
- package/dist/agent/actions.d.ts +32 -23
- package/dist/agent/actions.js +38 -24
- package/dist/agent/ai-sdk.js +5 -4
- package/dist/agent/error.js +3 -3
- package/dist/agent/invocation.js +3 -4
- package/dist/agent/model/sdk.js +2 -0
- package/dist/agent/model/tool-choice.d.ts +9 -0
- package/dist/agent/model/tool-choice.js +14 -0
- package/dist/agent/observation-feed.d.ts +5 -6
- package/dist/agent/observation-feed.js +8 -11
- package/dist/agent/observation.d.ts +40 -11
- package/dist/agent/observation.js +109 -59
- package/dist/agent/phases.d.ts +7 -1
- package/dist/agent/phases.js +12 -3
- package/dist/agent/point-tap.d.ts +8 -8
- package/dist/agent/point-tap.js +1 -1
- package/dist/agent/prompts.js +3 -0
- package/dist/agent/replay.d.ts +19 -8
- package/dist/agent/replay.js +29 -29
- package/dist/agent/scroll-target.d.ts +2 -3
- package/dist/agent/step-cache.d.ts +43 -15
- package/dist/agent/step-cache.js +156 -67
- package/dist/agent/tool-loop.js +94 -5
- package/dist/cache/anchors.d.ts +72 -23
- package/dist/cache/anchors.js +233 -60
- package/dist/cache/context.d.ts +25 -14
- package/dist/cache/context.js +50 -8
- package/dist/cache/decide.d.ts +2 -2
- package/dist/cache/decide.js +4 -6
- package/dist/cache/identity.d.ts +18 -4
- package/dist/cache/identity.js +9 -4
- package/dist/cache/recorder.d.ts +15 -1
- package/dist/cache/recorder.js +36 -23
- package/dist/cache/rekeyed.d.ts +46 -0
- package/dist/cache/rekeyed.js +89 -0
- package/dist/cache/relocate.d.ts +14 -14
- package/dist/cache/relocate.js +33 -25
- package/dist/cache/route.d.ts +32 -33
- package/dist/cache/route.js +94 -50
- package/dist/cache/store.d.ts +6 -0
- package/dist/cache/store.js +21 -1
- package/dist/cache/trace.d.ts +53 -14
- package/dist/cache/trace.js +76 -30
- package/dist/cli/bin.js +2 -2
- package/dist/cli/index.js +43 -32
- package/dist/cli/init/engines.d.ts +1 -3
- package/dist/cli/init/engines.js +2 -2
- package/dist/cli/init/gateways.d.ts +1 -1
- package/dist/cli/init/gateways.js +11 -1
- package/dist/cli/init/mcp-config.d.ts +2 -1
- package/dist/cli/init/mcp-config.js +8 -3
- package/dist/cli/init/sibling-versions.json +1 -1
- package/dist/cli/init/versions.d.ts +0 -8
- package/dist/cli/init/versions.js +1 -10
- package/dist/cli/mcp.d.ts +3 -2
- package/dist/cli/mcp.js +52 -16
- package/dist/cli/run-output.d.ts +17 -0
- package/dist/cli/run-output.js +45 -0
- package/dist/collect/collect.js +8 -4
- package/dist/collect/registry.js +27 -16
- package/dist/config/compiled-files.d.ts +36 -0
- package/dist/config/compiled-files.js +50 -0
- package/dist/config/diagnose.js +21 -19
- package/dist/config/esm-hooks.d.ts +39 -44
- package/dist/config/esm-hooks.js +238 -63
- package/dist/config/load.d.ts +1 -7
- package/dist/config/load.js +23 -66
- package/dist/config/register.d.ts +8 -0
- package/dist/config/register.js +9 -0
- package/dist/config/resolve.d.ts +1 -0
- package/dist/config/resolve.js +33 -1
- package/dist/config/specifiers.d.ts +19 -0
- package/dist/config/specifiers.js +33 -0
- package/dist/config/tsconfig.d.ts +56 -0
- package/dist/config/tsconfig.js +115 -0
- package/dist/config/typescript.d.ts +19 -0
- package/dist/config/typescript.js +253 -0
- package/dist/engine/contract.d.ts +1 -0
- package/dist/engine/index.d.ts +4 -2
- package/dist/engine/session.js +2 -2
- package/dist/expect/async.js +4 -2
- package/dist/expect/poll-scope.d.ts +52 -0
- package/dist/expect/poll-scope.js +113 -0
- package/dist/expect/poll.js +33 -6
- package/dist/expect/values.js +11 -4
- package/dist/internal/abandoned.d.ts +11 -0
- package/dist/internal/abandoned.js +23 -0
- package/dist/internal/brands.d.ts +2 -0
- package/dist/internal/brands.js +2 -0
- package/dist/internal/client-identity.d.ts +9 -0
- package/dist/internal/client-identity.js +14 -0
- package/dist/internal/errors.d.ts +10 -3
- package/dist/internal/errors.js +23 -8
- package/dist/internal/node-version.d.ts +16 -7
- package/dist/internal/node-version.js +50 -15
- package/dist/internal/package-manager.js +1 -1
- package/dist/internal/realm-slot.d.ts +5 -4
- package/dist/internal/redact.d.ts +9 -4
- package/dist/internal/redact.js +120 -23
- package/dist/internal/time.d.ts +26 -4
- package/dist/internal/time.js +58 -14
- package/dist/internal/urls.d.ts +9 -5
- package/dist/internal/urls.js +19 -10
- package/dist/locator/engine.d.ts +12 -2
- package/dist/locator/engine.js +55 -14
- package/dist/locator/screen.js +59 -9
- package/dist/mcp/catalog.d.ts +3 -2
- package/dist/mcp/catalog.js +6 -6
- package/dist/mcp/output.d.ts +59 -0
- package/dist/mcp/output.js +129 -0
- package/dist/mcp/server.d.ts +4 -0
- package/dist/mcp/server.js +9 -12
- package/dist/mcp/session.d.ts +8 -1
- package/dist/mcp/session.js +56 -15
- package/dist/mcp/tools.js +7 -5
- package/dist/oauth/chatgpt.js +3 -23
- package/dist/oauth/cli.js +1 -0
- package/dist/oauth/copilot.d.ts +5 -4
- package/dist/oauth/copilot.js +80 -7
- package/dist/oauth/grok.js +2 -1
- package/dist/oauth/models.js +2 -1
- package/dist/oauth/opencode-console.d.ts +10 -0
- package/dist/oauth/opencode-console.js +133 -0
- package/dist/oauth/providers/github-copilot.d.ts +12 -0
- package/dist/oauth/providers/github-copilot.js +77 -23
- package/dist/oauth/providers/opencode-console.d.ts +31 -0
- package/dist/oauth/providers/opencode-console.js +148 -0
- package/dist/oauth/providers.d.ts +2 -6
- package/dist/oauth/providers.js +3 -7
- package/dist/oauth/responses.d.ts +9 -0
- package/dist/oauth/responses.js +23 -0
- package/dist/report/build.d.ts +26 -0
- package/dist/report/build.js +16 -8
- package/dist/report/failure-text.d.ts +6 -1
- package/dist/report/failure-text.js +14 -4
- package/dist/report/format.d.ts +10 -5
- package/dist/report/format.js +12 -5
- package/dist/report/junit.js +11 -12
- package/dist/report/list.d.ts +12 -0
- package/dist/report/list.js +61 -24
- package/dist/report/markdown.d.ts +1 -1
- package/dist/report/markdown.js +37 -14
- package/dist/report/repeats.d.ts +6 -1
- package/dist/report/repeats.js +12 -3
- package/dist/run/artifacts.d.ts +22 -4
- package/dist/run/artifacts.js +71 -3
- package/dist/run/execute.d.ts +34 -5
- package/dist/run/execute.js +93 -44
- package/dist/run/extended-fixtures.js +6 -2
- package/dist/run/failure-evidence.js +10 -4
- package/dist/run/fixtures.js +6 -8
- package/dist/run/in-process.d.ts +2 -0
- package/dist/run/in-process.js +52 -10
- package/dist/run/last-run.d.ts +42 -8
- package/dist/run/last-run.js +121 -14
- package/dist/run/process-output.d.ts +73 -0
- package/dist/run/process-output.js +130 -0
- package/dist/run/provision.d.ts +2 -0
- package/dist/run/provision.js +1 -0
- package/dist/run/realm.d.ts +4 -1
- package/dist/run/realm.js +20 -6
- package/dist/run/records.d.ts +20 -4
- package/dist/run/records.js +26 -0
- package/dist/run/retry.d.ts +10 -1
- package/dist/run/retry.js +30 -12
- package/dist/run/runner.d.ts +9 -1
- package/dist/run/runner.js +101 -39
- package/dist/run/scheduler.js +255 -19
- package/dist/run/secrecy.d.ts +22 -19
- package/dist/run/secrecy.js +22 -18
- package/dist/run/serial.d.ts +75 -2
- package/dist/run/serial.js +243 -59
- package/dist/run/standalone.js +1 -1
- package/dist/run/steps.d.ts +0 -6
- package/dist/run/steps.js +3 -20
- package/dist/run/trace-redaction.d.ts +5 -6
- package/dist/run/trace-redaction.js +5 -6
- package/dist/run/worker/entry.js +40 -18
- package/dist/run/worker/handle.js +10 -5
- package/dist/run/worker/protocol.d.ts +85 -3
- package/dist/run/worker/session.d.ts +2 -0
- package/dist/run/worker/session.js +29 -0
- package/dist/telemetry/environment.d.ts +2 -1
- package/dist/telemetry/environment.js +12 -2
- package/dist/telemetry/events.js +1 -0
- package/dist/telemetry/project.d.ts +5 -6
- package/dist/telemetry/project.js +6 -11
- package/dist/telemetry/store.d.ts +5 -5
- package/dist/telemetry/store.js +5 -7
- package/dist/telemetry/telemetry.d.ts +2 -3
- package/dist/telemetry/telemetry.js +2 -4
- package/dist/types.d.ts +15 -6
- package/docs/agent-steps.mdx +1 -0
- package/docs/agents.mdx +7 -3
- package/docs/api-testing.mdx +2 -1
- package/docs/authentication.mdx +6 -3
- package/docs/browser.mdx +49 -12
- package/docs/bug-bash.mdx +2 -1
- package/docs/cache.mdx +66 -19
- package/docs/ci.mdx +14 -9
- package/docs/coding-agents.mdx +2 -1
- package/docs/core-concepts.mdx +208 -0
- package/docs/debugging.mdx +25 -12
- package/docs/decision-models.mdx +142 -0
- package/docs/example-projects.mdx +32 -0
- package/docs/executors.mdx +7 -2
- package/docs/explore.mdx +1 -0
- package/docs/github.mdx +14 -5
- package/docs/index.mdx +9 -5
- package/docs/integrations/eas.mdx +1 -0
- package/docs/integrations/index.mdx +1 -0
- package/docs/integrations/kernel.mdx +3 -2
- package/docs/migrate/cypress.mdx +26 -3
- package/docs/migrate/detox.mdx +24 -1
- package/docs/migrate/maestro.mdx +28 -4
- package/docs/migrate/playwright.mdx +44 -18
- package/docs/migrate/selenium.mdx +27 -3
- package/docs/mobile-ci.mdx +6 -5
- package/docs/mobile.mdx +96 -8
- package/docs/models.mdx +60 -9
- package/docs/quickstart.mdx +28 -16
- package/docs/reference/agent.mdx +13 -3
- package/docs/reference/app.mdx +2 -1
- package/docs/reference/cli.mdx +41 -18
- package/docs/reference/config.mdx +67 -13
- package/docs/reference/engine.mdx +25 -13
- package/docs/reference/environment.mdx +10 -2
- package/docs/reference/errors.mdx +13 -10
- package/docs/reference/expect.mdx +36 -13
- package/docs/reference/mcp.mdx +21 -8
- package/docs/reference/mobile.mdx +24 -13
- package/docs/reference/reporters.mdx +9 -2
- package/docs/reference/screen.mdx +49 -16
- package/docs/reference/test.mdx +32 -5
- package/docs/reference/web.mdx +133 -23
- package/docs/security.mdx +58 -42
- package/docs/subscriptions.mdx +92 -12
- package/docs/telemetry.mdx +9 -6
- package/docs/tools.mdx +1 -0
- package/docs/{starting-your-app.mdx → web.mdx} +157 -11
- package/docs/writing-an-engine.mdx +2 -1
- package/package.json +25 -8
- package/schema/report-v1.schema.json +14 -1
- package/skills/e2e/SKILL.md +5 -3
- package/skills/e2e/references/agent.md +33 -11
- package/skills/e2e/references/debugging.md +4 -4
- package/skills/e2e/references/mcp.md +8 -3
- package/skills/e2e/references/running.md +15 -9
- package/skills/e2e/references/setup.md +45 -16
- package/skills/e2e/references/writing-tests.md +35 -18
- package/docs/viewports.mdx +0 -146
- package/docs/writing-tests.mdx +0 -256
package/dist/agent/step-cache.js
CHANGED
|
@@ -11,17 +11,17 @@
|
|
|
11
11
|
* for the route precondition. The initial capture can also serve the executor
|
|
12
12
|
* when replay performed no action.
|
|
13
13
|
*/
|
|
14
|
-
import {
|
|
14
|
+
import { deltaEvidenced, deltaHolds, describeDelta } from '../cache/anchors.js';
|
|
15
15
|
import { decideTraceReplay, opensWithNavigate } from '../cache/decide.js';
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import { TraceRecorder } from '../cache/recorder.js';
|
|
16
|
+
import { sameRoute } from '../cache/route.js';
|
|
17
|
+
import { recordedProvenance, TraceRecorder } from '../cache/recorder.js';
|
|
19
18
|
import { expandTrace, templateParams, templatesCollide, templateTrace } from '../cache/template.js';
|
|
20
19
|
import { readTraceEntry } from '../cache/trace.js';
|
|
21
20
|
import { sleep } from '../internal/time.js';
|
|
22
21
|
import { AgentError, isAgentError, isModelUnreachable } from './error.js';
|
|
23
22
|
import { isRuntimeHardStop } from './executor.js';
|
|
24
|
-
import { replayTrace,
|
|
23
|
+
import { replayTrace, verifyEndState, } from './replay.js';
|
|
24
|
+
import { redactNodesAgain } from './observation.js';
|
|
25
25
|
/** The outcome of a step that threw `cause`. */
|
|
26
26
|
export function failedStepOutcome(cause) {
|
|
27
27
|
if (isModelUnreachable(cause))
|
|
@@ -48,9 +48,8 @@ const STALE_REASONS = new Set([
|
|
|
48
48
|
const END_WAIT_MARGIN_MS = 10_000;
|
|
49
49
|
/**
|
|
50
50
|
* How long a replay waits for a recorded destination path to be the current
|
|
51
|
-
* one.
|
|
52
|
-
*
|
|
53
|
-
* before the new document commits. Reading the path once, right after the
|
|
51
|
+
* one. The replayed tap that starts a navigation returns before the new
|
|
52
|
+
* document commits. Reading the path once, right after the
|
|
54
53
|
* tap, hands every navigation off as an end-mismatch; polling with the
|
|
55
54
|
* settling backoff lets the destination arrive. Bounded like anchor polling.
|
|
56
55
|
*/
|
|
@@ -60,8 +59,11 @@ export class StepTraceSession {
|
|
|
60
59
|
host;
|
|
61
60
|
cache;
|
|
62
61
|
keyHash;
|
|
62
|
+
/** The claimed key and the step it names, which an entry records as its provenance. */
|
|
63
|
+
claim;
|
|
63
64
|
recorder;
|
|
64
65
|
options;
|
|
66
|
+
redaction;
|
|
65
67
|
info;
|
|
66
68
|
prefix;
|
|
67
69
|
startPath;
|
|
@@ -85,6 +87,8 @@ export class StepTraceSession {
|
|
|
85
87
|
replayedWhole = false;
|
|
86
88
|
/** True once a cached entry's actions were run this step, fully or partly. */
|
|
87
89
|
consumedReplay = false;
|
|
90
|
+
/** True once the store returned an entry for this step, whether or not it replayed. */
|
|
91
|
+
readEntryHit = false;
|
|
88
92
|
/** True once `cache.strict` failed the step on its recording, which is then kept for review rather than evicted. */
|
|
89
93
|
failedStale = false;
|
|
90
94
|
/** Grammar actions recorded so far when an end-mismatch hand-off happened. */
|
|
@@ -92,13 +96,15 @@ export class StepTraceSession {
|
|
|
92
96
|
constructor(host, options) {
|
|
93
97
|
this.host = host;
|
|
94
98
|
this.options = options;
|
|
99
|
+
this.redaction = { redact: options.redact, redactCut: options.redactCut };
|
|
95
100
|
this.cache = options.cache;
|
|
96
101
|
// The key digests the params as the recording spells them, a placeholder
|
|
97
102
|
// where each `unique()` value was, so every run's value finds one entry.
|
|
98
|
-
this.
|
|
103
|
+
this.claim = options.cache.claimKey('act', options.instruction, templateParams(options.params, options.templates), options.agent);
|
|
104
|
+
this.keyHash = this.claim.keyHash;
|
|
99
105
|
if (options.cache.mode === 'read-write') {
|
|
100
106
|
this.recorder = new TraceRecorder({
|
|
101
|
-
|
|
107
|
+
...this.redaction,
|
|
102
108
|
maxActions: options.maxActions,
|
|
103
109
|
});
|
|
104
110
|
}
|
|
@@ -130,7 +136,8 @@ export class StepTraceSession {
|
|
|
130
136
|
* self-finalized verdict on a full replay whose postcondition holds;
|
|
131
137
|
* undefined dispatches the executor — after a miss from the top, after a
|
|
132
138
|
* divergence mid-step with `replayedPrefix` set. Every failure to replay is
|
|
133
|
-
* a miss, never an error; only runtime hard stops
|
|
139
|
+
* a miss, never an error; only runtime hard stops and a policy denial on
|
|
140
|
+
* the opening look propagate.
|
|
134
141
|
*/
|
|
135
142
|
async begin() {
|
|
136
143
|
this.startedMs = Date.now();
|
|
@@ -144,6 +151,8 @@ export class StepTraceSession {
|
|
|
144
151
|
this.info = this.missed(read.reason, 0);
|
|
145
152
|
if (read.unavailable !== true)
|
|
146
153
|
this.failIfStale();
|
|
154
|
+
if (read.reason === 'no-entry')
|
|
155
|
+
await this.failIfRekeyed();
|
|
147
156
|
return undefined;
|
|
148
157
|
}
|
|
149
158
|
// With an entry in hand the step is the cache's from its first moment: the
|
|
@@ -151,6 +160,7 @@ export class StepTraceSession {
|
|
|
151
160
|
// replay finishes stays the cache's through its verdict and the re-stage,
|
|
152
161
|
// so a reporter never shows a model turn that is not coming. Only a replay
|
|
153
162
|
// that cannot finish the step hands it to the model.
|
|
163
|
+
this.readEntryHit = true;
|
|
154
164
|
this.host.replaying(true);
|
|
155
165
|
const verdict = await this.replayEntry(read.entry);
|
|
156
166
|
if (verdict === undefined) {
|
|
@@ -173,6 +183,23 @@ export class StepTraceSession {
|
|
|
173
183
|
this.failedStale = true;
|
|
174
184
|
throw new AgentError('REPLAY_STALE', `the recording of this step no longer replays (${reason}), and cache.strict hands no step to the agent; ${strict.advice}`);
|
|
175
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* Under `cache.strict`, ends a step whose key found no entry while the
|
|
188
|
+
* store holds a recording made for the same step under another key: the
|
|
189
|
+
* runner, the engine, the app, or the agent's context changed since, and
|
|
190
|
+
* the recording no longer replays as surely as one that diverged. A step
|
|
191
|
+
* whose instruction or params changed is a new step and still runs live.
|
|
192
|
+
*/
|
|
193
|
+
async failIfRekeyed() {
|
|
194
|
+
const { strict } = this.cache;
|
|
195
|
+
if (strict === false || strict.recordings === undefined)
|
|
196
|
+
return;
|
|
197
|
+
const previous = await strict.recordings.underAnotherKey(this.keyHash, recordedProvenance(this.claim.step, this.options.redact));
|
|
198
|
+
if (previous === undefined)
|
|
199
|
+
return;
|
|
200
|
+
this.failedStale = true;
|
|
201
|
+
throw new AgentError('REPLAY_STALE', `the recording of this step no longer replays: the store holds it under another cache key (${previous}.json), since the runner, the engine, the app, or the agent's context changed after it was recorded, and cache.strict hands no step to the agent; ${strict.advice}`);
|
|
202
|
+
}
|
|
176
203
|
/**
|
|
177
204
|
* Closes the session once the step has settled. The whole write-side
|
|
178
205
|
* decision lives here, in read-write mode only:
|
|
@@ -192,6 +219,10 @@ export class StepTraceSession {
|
|
|
192
219
|
* anchor, or end wait, so drift is repaired only once a relocation fails
|
|
193
220
|
* and the hand-off that follows re-records.
|
|
194
221
|
* - passed otherwise: stage the recorded trace for attempt-end settlement.
|
|
222
|
+
* When this pass leaves nothing to stage (it changed nothing a replay
|
|
223
|
+
* could check) and an entry was read for the step, evict that entry: it
|
|
224
|
+
* did not serve this pass, and nothing recorded will replace it, so
|
|
225
|
+
* keeping it would hand every later run off the same way.
|
|
195
226
|
* - failed after consuming a replay: evict. Without this, a diverged replay
|
|
196
227
|
* whose step then fails stages nothing — and the poisoned entry would
|
|
197
228
|
* replay its bad prefix on every future first attempt.
|
|
@@ -219,10 +250,15 @@ export class StepTraceSession {
|
|
|
219
250
|
if (this.repairedAfterEndMismatch(recorder))
|
|
220
251
|
await this.evict();
|
|
221
252
|
else if (this.replayedWhole) {
|
|
222
|
-
this.cache.staged.push({
|
|
253
|
+
this.cache.staged.push({
|
|
254
|
+
kind: 'keep',
|
|
255
|
+
keyHash: this.keyHash,
|
|
256
|
+
stepIndex: this.options.stepIndex,
|
|
257
|
+
recordedFor: recordedProvenance(this.claim.step, this.options.redact),
|
|
258
|
+
});
|
|
223
259
|
}
|
|
224
|
-
else
|
|
225
|
-
await this.
|
|
260
|
+
else if (!(await this.stage(recorder, verdictSummary)) && this.readEntryHit)
|
|
261
|
+
await this.evict();
|
|
226
262
|
return;
|
|
227
263
|
}
|
|
228
264
|
}
|
|
@@ -262,7 +298,7 @@ export class StepTraceSession {
|
|
|
262
298
|
* when this step may write. Settled as far as the capture's purpose asks.
|
|
263
299
|
*/
|
|
264
300
|
async captureStart(purpose) {
|
|
265
|
-
const observation = await probeScreen(this.host, purpose === 'path-only' ? 'raw' : 'held-still');
|
|
301
|
+
const observation = await probeScreen(this.host, purpose === 'path-only' ? 'raw' : 'held-still', true);
|
|
266
302
|
this.startPath = observation?.path;
|
|
267
303
|
if (this.recorder !== undefined && observation?.kind === 'semantic')
|
|
268
304
|
this.startNodes = observation.nodes;
|
|
@@ -289,10 +325,33 @@ export class StepTraceSession {
|
|
|
289
325
|
this.info = this.missed(decision.reason, trace.actions.length);
|
|
290
326
|
return undefined;
|
|
291
327
|
}
|
|
292
|
-
|
|
328
|
+
// Every screen the replay looks at, in order, from the start: the end
|
|
329
|
+
// check measures the recorded delta from the first of them on the route
|
|
330
|
+
// the replay ends on, so a free action is preceded by a look until one
|
|
331
|
+
// on the recorded end route has been seen.
|
|
332
|
+
const screens = start === undefined ? [] : [start];
|
|
333
|
+
const onEndRoute = (screen) => screen !== undefined && (trace.endPath === undefined || (screen.path !== undefined && sameRoute(screen.path, trace.endPath)));
|
|
334
|
+
const host = this.host;
|
|
335
|
+
const watched = {
|
|
336
|
+
get traceEligible() {
|
|
337
|
+
return host.traceEligible;
|
|
338
|
+
},
|
|
339
|
+
observe: async (mode) => {
|
|
340
|
+
const screen = await host.observe(mode);
|
|
341
|
+
screens.push(screen);
|
|
342
|
+
return screen;
|
|
343
|
+
},
|
|
344
|
+
actions: host.actions,
|
|
345
|
+
signal: host.signal,
|
|
346
|
+
remainingMs: () => host.remainingMs(),
|
|
347
|
+
};
|
|
348
|
+
const outcome = await replayTrace(watched, trace, {
|
|
349
|
+
...(start?.kind === 'semantic' ? { initial: start } : {}),
|
|
350
|
+
looksBeforeFree: () => !onEndRoute(screens.at(-1)),
|
|
351
|
+
});
|
|
293
352
|
this.consumedReplay = true;
|
|
294
353
|
const stopReason = outcome.completed
|
|
295
|
-
? (await this.endStateMatches(trace))
|
|
354
|
+
? start?.kind === 'semantic' && (await this.endStateMatches(trace, start, screens))
|
|
296
355
|
? undefined
|
|
297
356
|
: 'end-mismatch'
|
|
298
357
|
: (outcome.stopReason ?? 'action-failed');
|
|
@@ -309,57 +368,59 @@ export class StepTraceSession {
|
|
|
309
368
|
}
|
|
310
369
|
/**
|
|
311
370
|
* The trace's postcondition against the live screen: the recorded end
|
|
312
|
-
* route
|
|
313
|
-
*
|
|
314
|
-
*
|
|
371
|
+
* route when the live location is known, the recorded delta on the
|
|
372
|
+
* screen (`deltaHolds`), and evidence that the replay produced it rather
|
|
373
|
+
* than finding it there (`deltaEvidenced`). The evidence is measured from
|
|
374
|
+
* the first screen the replay saw on the route it ended on (`screens`, in
|
|
375
|
+
* order): the start for a step that stays on one screen, the page a
|
|
376
|
+
* recorded navigate opened before the switch on it was tapped. An outcome
|
|
377
|
+
* already showing there proves nothing, and neither does a recording with
|
|
378
|
+
* no delta at all.
|
|
315
379
|
*/
|
|
316
|
-
async endStateMatches(trace) {
|
|
380
|
+
async endStateMatches(trace, start, screens) {
|
|
317
381
|
if (!this.host.traceEligible)
|
|
318
382
|
return false;
|
|
319
383
|
const arrived = await this.endScreen(trace);
|
|
320
384
|
if (arrived === undefined)
|
|
321
385
|
return false;
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
386
|
+
// Both captures may predate a secret the replay resolved; read with the
|
|
387
|
+
// ledger as it is now, an unchanged node is no change.
|
|
388
|
+
const baseline = baselineScreen(screens, arrived.path);
|
|
389
|
+
const baselineNodes = baseline === undefined ? undefined : redactNodesAgain(baseline.nodes, this.redaction);
|
|
390
|
+
if (!deltaEvidenced(trace, baselineNodes, inputTargets(trace)))
|
|
391
|
+
return false;
|
|
392
|
+
const beforeNodes = baselineNodes ?? redactNodesAgain(start.nodes, this.redaction);
|
|
393
|
+
// Every look the wait takes must still be on the recorded end route: a
|
|
394
|
+
// screen that moved on after the route first matched is another screen.
|
|
395
|
+
const holds = (screen) => (trace.endPath === undefined || screen.path === undefined || sameRoute(screen.path, trace.endPath)) &&
|
|
396
|
+
deltaHolds(trace, screen.nodes, beforeNodes);
|
|
397
|
+
return (await verifyEndState(this.host, holds, {
|
|
398
|
+
initial: arrived,
|
|
326
399
|
...(trace.endWaitMs === undefined ? {} : { waitMs: trace.endWaitMs }),
|
|
327
400
|
})) && this.host.traceEligible;
|
|
328
401
|
}
|
|
329
402
|
/**
|
|
330
403
|
* Captures the semantic end state once its route matches the recording,
|
|
331
|
-
* polling while a navigation the last action started commits.
|
|
332
|
-
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
335
|
-
* the recording, is the one that decides it, as for a route that matched.
|
|
404
|
+
* polling while a navigation the last action started commits. A route
|
|
405
|
+
* that never matches is another screen, whatever it shows: a link that
|
|
406
|
+
* now lands on a page sharing the recorded one's layout is exactly the
|
|
407
|
+
* flow that went wrong.
|
|
336
408
|
*/
|
|
337
409
|
async endScreen(trace) {
|
|
338
410
|
const startedMs = Date.now();
|
|
339
|
-
const recorded = trace.endPath
|
|
340
|
-
const anchors = trace.endAnchors ?? [];
|
|
411
|
+
const recorded = trace.endPath;
|
|
341
412
|
for (let attempt = 0;; attempt += 1) {
|
|
342
413
|
const observation = await probeScreen(this.host, 'raw');
|
|
343
414
|
if (observation?.kind !== 'semantic' || !this.host.traceEligible)
|
|
344
415
|
return undefined;
|
|
345
|
-
if (recorded === undefined || observation.path === undefined)
|
|
346
|
-
return
|
|
347
|
-
const verdict = compareRoutes(recorded, routeOf(observation.path));
|
|
348
|
-
if (verdict === 'same')
|
|
349
|
-
return { screen: observation, anchorsSeen: false };
|
|
350
|
-
// A path the runner cannot recognize as the recorded route is the
|
|
351
|
-
// recorded screen only if the recorded effect is visibly on it; with
|
|
352
|
-
// no anchors recorded there is nothing to see, and it is another screen.
|
|
353
|
-
const undecided = verdict === 'undecided' && anchors.length > 0;
|
|
354
|
-
if (undecided && anchorsPresent(anchors, observation.nodes, { redact: this.options.redact })) {
|
|
355
|
-
return { screen: observation, anchorsSeen: true };
|
|
356
|
-
}
|
|
416
|
+
if (recorded === undefined || observation.path === undefined || sameRoute(recorded, observation.path))
|
|
417
|
+
return observation;
|
|
357
418
|
const delay = END_PATH_DELAYS_MS[attempt];
|
|
358
419
|
if (delay === undefined ||
|
|
359
420
|
Date.now() - startedMs + delay > END_PATH_TIMEOUT_MS ||
|
|
360
421
|
this.host.remainingMs() <= delay ||
|
|
361
422
|
this.host.signal.aborted) {
|
|
362
|
-
return
|
|
423
|
+
return undefined;
|
|
363
424
|
}
|
|
364
425
|
await sleep(delay, this.host.signal);
|
|
365
426
|
}
|
|
@@ -401,32 +462,36 @@ export class StepTraceSession {
|
|
|
401
462
|
*
|
|
402
463
|
* The end path and a fresh settled observation are the trace's
|
|
403
464
|
* postcondition — the state the step passed in. The delta between the
|
|
404
|
-
* baseline and the passing observation becomes the
|
|
405
|
-
* step moved to another pathname the whole new
|
|
406
|
-
* first stable anchors back the path check,
|
|
407
|
-
* record's page up to the id the app minted for
|
|
408
|
-
* cannot be captured stages nothing
|
|
409
|
-
* replay
|
|
465
|
+
* baseline and the passing observation becomes the anchors: what appeared
|
|
466
|
+
* and what vanished. When the step moved to another pathname the whole new
|
|
467
|
+
* screen is the delta, and its first stable anchors back the path check,
|
|
468
|
+
* which matches a created record's page up to the id the app minted for
|
|
469
|
+
* it. A postcondition that cannot be captured stages nothing, and neither
|
|
470
|
+
* does a step that changed nothing a replay could check, no node and no
|
|
471
|
+
* route: a trace without its check would replay on mechanics alone.
|
|
410
472
|
*/
|
|
411
473
|
async stage(recorder, verdictSummary) {
|
|
412
474
|
if (!this.host.traceEligible || this.startNodes === undefined || recorder.recordedCount === 0)
|
|
413
|
-
return;
|
|
475
|
+
return false;
|
|
414
476
|
const observation = await probeScreen(this.host, 'held-still');
|
|
415
477
|
if (!this.host.traceEligible || observation?.kind !== 'semantic')
|
|
416
|
-
return;
|
|
478
|
+
return false;
|
|
417
479
|
const { nodes: endNodes, path: endPath } = observation;
|
|
418
|
-
|
|
480
|
+
// The start capture may predate a secret this step resolved; read with
|
|
481
|
+
// the ledger as it is now, an unchanged node is no delta.
|
|
482
|
+
const startNodes = redactNodesAgain(this.startNodes, this.redaction);
|
|
483
|
+
const routeMoved = this.startPath !== undefined && endPath !== undefined && !sameRoute(this.startPath, endPath);
|
|
484
|
+
const delta = describeDelta(startNodes, endNodes, routeMoved);
|
|
485
|
+
if (delta.appeared.length === 0 && delta.gone.length === 0 && !routeMoved)
|
|
486
|
+
return false;
|
|
419
487
|
const trace = recorder.finalize({
|
|
420
488
|
executor: this.options.executor,
|
|
421
|
-
recordedFor:
|
|
422
|
-
testId: this.cache.identity.testId,
|
|
423
|
-
targetId: this.cache.identity.targetId,
|
|
424
|
-
instructionDigest: instructionDigest(this.options.instruction),
|
|
425
|
-
},
|
|
489
|
+
recordedFor: this.claim.step,
|
|
426
490
|
summary: verdictSummary ?? 'step passed',
|
|
427
491
|
...(this.startPath === undefined ? {} : { startPath: this.startPath }),
|
|
428
492
|
...(endPath === undefined ? {} : { endPath }),
|
|
429
|
-
endAnchors,
|
|
493
|
+
endAnchors: delta.appeared,
|
|
494
|
+
goneAnchors: delta.gone,
|
|
430
495
|
// How long the app took to show its end state after the last action,
|
|
431
496
|
// plus room for a slower day: the budget a replay waits for the anchors
|
|
432
497
|
// to return. Measured from the last action, not the step's start: the
|
|
@@ -435,12 +500,12 @@ export class StepTraceSession {
|
|
|
435
500
|
endWaitMs: Date.now() - (recorder.lastActionAtMs ?? this.startedMs) + END_WAIT_MARGIN_MS,
|
|
436
501
|
});
|
|
437
502
|
if (trace === undefined)
|
|
438
|
-
return;
|
|
503
|
+
return false;
|
|
439
504
|
// A trace with no start anchor — no recorded path (a surface without a URL)
|
|
440
505
|
// and no opening navigate — could never replay: `wrong-context` forever.
|
|
441
506
|
// Writing it would be pure store traffic, so it is not written at all.
|
|
442
507
|
if (trace.startPath === undefined && !opensWithNavigate(trace))
|
|
443
|
-
return;
|
|
508
|
+
return false;
|
|
444
509
|
// Stored with a slot where each `unique()` value appeared, so the next
|
|
445
510
|
// run's values — a fresh timestamped name — replay the same flow. A
|
|
446
511
|
// recording that cannot be templated safely is not written at all; when
|
|
@@ -448,12 +513,13 @@ export class StepTraceSession {
|
|
|
448
513
|
if (templatesCollide(this.options.params, this.options.templates)) {
|
|
449
514
|
if (this.info !== undefined)
|
|
450
515
|
this.info = { ...this.info, notRecorded: 'param-collision' };
|
|
451
|
-
return;
|
|
516
|
+
return false;
|
|
452
517
|
}
|
|
453
518
|
const templated = templateTrace(trace, this.options.templates);
|
|
454
519
|
if (templated === undefined)
|
|
455
|
-
return;
|
|
520
|
+
return false;
|
|
456
521
|
this.cache.staged.push({ kind: 'write', keyHash: this.keyHash, trace: templated, stepIndex: this.options.stepIndex });
|
|
522
|
+
return true;
|
|
457
523
|
}
|
|
458
524
|
/**
|
|
459
525
|
* Awaited so the step does not close — and a later read cannot be served
|
|
@@ -476,18 +542,41 @@ export class StepTraceSession {
|
|
|
476
542
|
};
|
|
477
543
|
}
|
|
478
544
|
}
|
|
545
|
+
/** The controls a trace's input actions set a value or a state on: their changed anchors echo the input. */
|
|
546
|
+
function inputTargets(trace) {
|
|
547
|
+
return trace.actions.flatMap((action) => ('target' in action && action.target !== undefined ? [action.target] : []));
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* The first semantic screen of the replay's last stretch on the route it
|
|
551
|
+
* ended on (`endPath`), or undefined when its last screen before the end
|
|
552
|
+
* was on another route, so the last action moved it. A screen or an end
|
|
553
|
+
* without a location counts as on the route: there is nothing to compare.
|
|
554
|
+
*/
|
|
555
|
+
function baselineScreen(screens, endPath) {
|
|
556
|
+
let baseline;
|
|
557
|
+
for (const screen of screens) {
|
|
558
|
+
if (endPath !== undefined && screen.path !== undefined && !sameRoute(screen.path, endPath))
|
|
559
|
+
baseline = undefined;
|
|
560
|
+
else if (screen.kind === 'semantic')
|
|
561
|
+
baseline ??= screen;
|
|
562
|
+
}
|
|
563
|
+
return baseline;
|
|
564
|
+
}
|
|
479
565
|
/**
|
|
480
566
|
* One cache capture, settled as far as `mode` asks, or undefined when the
|
|
481
567
|
* surface cannot be observed right now, which is the executor's business,
|
|
482
568
|
* not the cache's. Runtime hard stops (timeout, cancellation) are the step's
|
|
483
|
-
* truth even when they land during cache bookkeeping, and propagate.
|
|
569
|
+
* truth even when they land during cache bookkeeping, and propagate. So does
|
|
570
|
+
* a policy denial on the step's opening look (`opening`): the executor's
|
|
571
|
+
* first look reads the same screen under the same policy, and would spend
|
|
572
|
+
* another operation budget to be denied again.
|
|
484
573
|
*/
|
|
485
|
-
async function probeScreen(host, mode) {
|
|
574
|
+
async function probeScreen(host, mode, opening = false) {
|
|
486
575
|
try {
|
|
487
576
|
return await host.observe(mode);
|
|
488
577
|
}
|
|
489
578
|
catch (cause) {
|
|
490
|
-
if (isRuntimeHardStop(cause))
|
|
579
|
+
if (isRuntimeHardStop(cause) || (opening && isAgentError(cause) && cause.code === 'POLICY_DENIED'))
|
|
491
580
|
throw cause;
|
|
492
581
|
return undefined;
|
|
493
582
|
}
|
package/dist/agent/tool-loop.js
CHANGED
|
@@ -11,10 +11,11 @@
|
|
|
11
11
|
* - model-call accounting and a step transcript for `--debug`.
|
|
12
12
|
*/
|
|
13
13
|
import { asSdkLanguageModel, loadAiSdk } from './ai-sdk.js';
|
|
14
|
+
import { MODEL_REQUEST_HEADERS } from '../internal/client-identity.js';
|
|
14
15
|
import { ConfigurationError, withHint } from '../internal/errors.js';
|
|
15
16
|
import { failureHint, isAbort, TRANSPORT_RETRIES } from './model/sdk.js';
|
|
16
17
|
import { isContextOverflow } from './model/overflow.js';
|
|
17
|
-
import { isForcedToolCallSkipped, isForcedToolChoiceDowngraded, isForcedToolChoiceRejected } from './model/tool-choice.js';
|
|
18
|
+
import { isForcedToolCallMismatched, isForcedToolCallSkipped, isForcedToolChoiceDowngraded, isForcedToolChoiceRejected, } from './model/tool-choice.js';
|
|
18
19
|
import { withStallGuard } from './model/stall.js';
|
|
19
20
|
import { providerHints } from './model/provider-hints.js';
|
|
20
21
|
import { isScreenOutput, toolResultTexts } from './screen-update.js';
|
|
@@ -192,6 +193,13 @@ class LoopRun {
|
|
|
192
193
|
prompt = shrunk;
|
|
193
194
|
continue;
|
|
194
195
|
}
|
|
196
|
+
const corrected = this.correctForcedTurn(cause, tracker);
|
|
197
|
+
if (corrected === 'out-of-turns')
|
|
198
|
+
break;
|
|
199
|
+
if (corrected !== undefined) {
|
|
200
|
+
prompt = corrected;
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
195
203
|
const freed = this.freeToolChoiceRetry(cause, tracker);
|
|
196
204
|
if (freed !== undefined) {
|
|
197
205
|
prompt = freed;
|
|
@@ -241,6 +249,7 @@ class LoopRun {
|
|
|
241
249
|
toolChoice: this.toolChoice,
|
|
242
250
|
...(providerOptions === undefined ? {} : { providerOptions: providerOptions }),
|
|
243
251
|
maxRetries: TRANSPORT_RETRIES,
|
|
252
|
+
headers: MODEL_REQUEST_HEADERS,
|
|
244
253
|
stopWhen: [
|
|
245
254
|
() => this.conclusion.concluded() || this.hardStop !== undefined,
|
|
246
255
|
// Counted across every generate call of the step, so a retry after an
|
|
@@ -273,6 +282,49 @@ class LoopRun {
|
|
|
273
282
|
{ role: 'user', content: TEXT_REPLY_NOTICE },
|
|
274
283
|
];
|
|
275
284
|
}
|
|
285
|
+
/**
|
|
286
|
+
* A turn forced to call `complete_step` that called other tools instead:
|
|
287
|
+
* the SDK throws `ToolChoiceViolationError` before running any of them. It
|
|
288
|
+
* is handled like a call to a tool the turn does not offer: the turn counts,
|
|
289
|
+
* each call goes back to the model as an error result, and the model is
|
|
290
|
+
* asked again on the same turn budget. Out of turns, the step ends as
|
|
291
|
+
* `STEP_NO_CONCLUSION`, like any step that never concluded.
|
|
292
|
+
*/
|
|
293
|
+
correctForcedTurn(cause, tracker) {
|
|
294
|
+
if (this.lastRequest === undefined || this.context.signal.aborted || this.hardStop !== undefined)
|
|
295
|
+
return undefined;
|
|
296
|
+
if (!isForcedToolCallMismatched(cause))
|
|
297
|
+
return undefined;
|
|
298
|
+
const reply = thrownReply(cause.content);
|
|
299
|
+
const results = reply.calls.map((call) => ({
|
|
300
|
+
type: 'tool-result',
|
|
301
|
+
toolCallId: call.toolCallId,
|
|
302
|
+
toolName: call.toolName,
|
|
303
|
+
output: {
|
|
304
|
+
type: 'error-text',
|
|
305
|
+
value: `${call.toolName} did not run: only complete_step is available now. Call complete_step with your verdict.`,
|
|
306
|
+
},
|
|
307
|
+
}));
|
|
308
|
+
this.recordThrownTurn(tracker, reply.calls.map((call) => `${call.toolName}(${truncate(safeJson(call.input), MAX_TURN_CALL_CHARS)})`), [
|
|
309
|
+
skippedReplyText(cause.content),
|
|
310
|
+
...results.map((result) => `[${result.toolName}] error: ${result.output.value}`),
|
|
311
|
+
].filter((line) => line !== '').join('\n'));
|
|
312
|
+
if (this.turnsUsed >= this.maxTurns)
|
|
313
|
+
return 'out-of-turns';
|
|
314
|
+
this.note(`turn ${String(this.turnsUsed)} called other tools in place of complete_step: asking again`);
|
|
315
|
+
this.turnOffset = this.turnsUsed;
|
|
316
|
+
return [...this.lastRequest, reply.message, { role: 'tool', content: results }];
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Counts a turn the provider answered, and billed, before the SDK threw on
|
|
320
|
+
* it. The SDK reports no usage for such a turn.
|
|
321
|
+
*/
|
|
322
|
+
recordThrownTurn(tracker, calls, outcome) {
|
|
323
|
+
tracker.onStepEnd({ usage: UNREPORTED_USAGE, providerMetadata: undefined, reasoningText: undefined });
|
|
324
|
+
this.turnsUsed += 1;
|
|
325
|
+
this.turns.push({ index: this.turnsUsed, calls, outcome, notes: [] });
|
|
326
|
+
this.attachTranscript();
|
|
327
|
+
}
|
|
276
328
|
/**
|
|
277
329
|
* A model that refuses a forced tool choice (HTTP 400 naming
|
|
278
330
|
* `tool_choice`) is asked again with `auto` and a tool-calls-only rule in
|
|
@@ -288,10 +340,7 @@ class LoopRun {
|
|
|
288
340
|
if (this.context.signal.aborted || this.hardStop !== undefined)
|
|
289
341
|
return undefined;
|
|
290
342
|
if (isForcedToolCallSkipped(cause)) {
|
|
291
|
-
|
|
292
|
-
tracker.onStepEnd({ usage: UNREPORTED_USAGE, providerMetadata: undefined, reasoningText: undefined });
|
|
293
|
-
this.turnsUsed += 1;
|
|
294
|
-
this.turns.push({ index: this.turnsUsed, calls: [], outcome: skippedReplyText(cause.content), notes: [] });
|
|
343
|
+
this.recordThrownTurn(tracker, [], skippedReplyText(cause.content));
|
|
295
344
|
this.note(`turn ${String(this.turnsUsed)} answered without the forced tool call: retrying with auto`);
|
|
296
345
|
}
|
|
297
346
|
else if (isForcedToolChoiceRejected(cause)) {
|
|
@@ -617,6 +666,46 @@ function skippedReplyText(content) {
|
|
|
617
666
|
.trim();
|
|
618
667
|
return text === '' ? '' : `assistant: ${truncate(text, MAX_TURN_TEXT_CHARS)}`;
|
|
619
668
|
}
|
|
669
|
+
/**
|
|
670
|
+
* A reply the SDK threw on, as the assistant message the SDK's own history
|
|
671
|
+
* would carry: its text, reasoning, and tool calls, each with the provider's
|
|
672
|
+
* metadata (reasoning signatures and Gemini's thought signatures ride there),
|
|
673
|
+
* and the tool calls apart, to answer each with a result.
|
|
674
|
+
*/
|
|
675
|
+
function thrownReply(content) {
|
|
676
|
+
const parts = [];
|
|
677
|
+
const calls = [];
|
|
678
|
+
for (const part of content) {
|
|
679
|
+
const metadata = part.providerMetadata === undefined ? {} : { providerOptions: part.providerMetadata };
|
|
680
|
+
if (part.type === 'text' && typeof part.text === 'string' && part.text !== '') {
|
|
681
|
+
parts.push({ type: 'text', text: part.text, ...metadata });
|
|
682
|
+
}
|
|
683
|
+
else if (part.type === 'reasoning' && typeof part.text === 'string') {
|
|
684
|
+
parts.push({ type: 'reasoning', text: part.text, ...metadata });
|
|
685
|
+
}
|
|
686
|
+
else if (part.type === 'tool-call' && typeof part.toolCallId === 'string' && typeof part.toolName === 'string') {
|
|
687
|
+
const call = {
|
|
688
|
+
type: 'tool-call',
|
|
689
|
+
toolCallId: part.toolCallId,
|
|
690
|
+
toolName: part.toolName,
|
|
691
|
+
input: typeof part.input === 'string' ? parseToolInput(part.input) : part.input,
|
|
692
|
+
...metadata,
|
|
693
|
+
};
|
|
694
|
+
parts.push(call);
|
|
695
|
+
calls.push(call);
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
return { message: { role: 'assistant', content: parts }, calls };
|
|
699
|
+
}
|
|
700
|
+
/** A raw tool call's input for the history: the parsed JSON, or the text itself when it does not parse. */
|
|
701
|
+
function parseToolInput(input) {
|
|
702
|
+
try {
|
|
703
|
+
return JSON.parse(input);
|
|
704
|
+
}
|
|
705
|
+
catch {
|
|
706
|
+
return input;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
620
709
|
function describeOutput(output) {
|
|
621
710
|
if (isScreenOutput(output))
|
|
622
711
|
return `${output.text}\n[screenshot, ${String(output.pixels.data.byteLength)} bytes]`;
|