@e2edev/e2e 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/agent/act-validation.d.ts +7 -4
- package/dist/agent/act-validation.d.ts.map +1 -1
- package/dist/agent/act-validation.js +18 -3
- package/dist/agent/act-validation.js.map +1 -1
- package/dist/agent/act.d.ts +2 -2
- package/dist/agent/act.d.ts.map +1 -1
- package/dist/agent/act.js +228 -51
- package/dist/agent/act.js.map +1 -1
- package/dist/agent/default-agent.d.ts +1 -1
- package/dist/agent/default-agent.d.ts.map +1 -1
- package/dist/agent/default-agent.js +19 -84
- package/dist/agent/default-agent.js.map +1 -1
- package/dist/agent/executor.d.ts +7 -0
- package/dist/agent/executor.d.ts.map +1 -1
- package/dist/agent/executor.js.map +1 -1
- package/dist/agent/index.d.ts.map +1 -1
- package/dist/agent/index.js +14 -6
- package/dist/agent/index.js.map +1 -1
- package/dist/agent/invocation.d.ts +1 -8
- package/dist/agent/invocation.d.ts.map +1 -1
- package/dist/agent/invocation.js +7 -51
- package/dist/agent/invocation.js.map +1 -1
- package/dist/agent/ledger.d.ts.map +1 -1
- package/dist/agent/ledger.js +10 -1
- package/dist/agent/ledger.js.map +1 -1
- package/dist/agent/loop-guards.d.ts +21 -0
- package/dist/agent/loop-guards.d.ts.map +1 -1
- package/dist/agent/loop-guards.js +48 -0
- package/dist/agent/loop-guards.js.map +1 -1
- package/dist/agent/model/sdk.d.ts +18 -0
- package/dist/agent/model/sdk.d.ts.map +1 -1
- package/dist/agent/model/sdk.js +9 -3
- package/dist/agent/model/sdk.js.map +1 -1
- package/dist/agent/observation-budget.d.ts +27 -0
- package/dist/agent/observation-budget.d.ts.map +1 -0
- package/dist/agent/observation-budget.js +43 -0
- package/dist/agent/observation-budget.js.map +1 -0
- package/dist/agent/observation.d.ts +40 -7
- package/dist/agent/observation.d.ts.map +1 -1
- package/dist/agent/observation.js +56 -11
- package/dist/agent/observation.js.map +1 -1
- package/dist/agent/phases.d.ts +1 -1
- package/dist/agent/phases.d.ts.map +1 -1
- package/dist/agent/phases.js +11 -13
- package/dist/agent/phases.js.map +1 -1
- package/dist/agent/primitives.d.ts +14 -2
- package/dist/agent/primitives.d.ts.map +1 -1
- package/dist/agent/primitives.js +98 -54
- package/dist/agent/primitives.js.map +1 -1
- package/dist/agent/screen-update.d.ts +47 -0
- package/dist/agent/screen-update.d.ts.map +1 -0
- package/dist/agent/screen-update.js +222 -0
- package/dist/agent/screen-update.js.map +1 -0
- package/dist/agent/step-cache.d.ts +38 -10
- package/dist/agent/step-cache.d.ts.map +1 -1
- package/dist/agent/step-cache.js +86 -45
- package/dist/agent/step-cache.js.map +1 -1
- package/dist/agent/tool-loop.d.ts.map +1 -1
- package/dist/agent/tool-loop.js +27 -4
- package/dist/agent/tool-loop.js.map +1 -1
- package/dist/cache/decide.d.ts +4 -3
- package/dist/cache/decide.d.ts.map +1 -1
- package/dist/cache/decide.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init/engines.d.ts +2 -3
- package/dist/cli/init/engines.d.ts.map +1 -1
- package/dist/cli/init/engines.js +3 -4
- package/dist/cli/init/engines.js.map +1 -1
- package/dist/cli/signals.d.ts +3 -3
- package/dist/cli/signals.js +3 -3
- package/dist/config/agent.d.ts +6 -2
- package/dist/config/agent.d.ts.map +1 -1
- package/dist/config/agent.js +6 -2
- package/dist/config/agent.js.map +1 -1
- package/dist/config/app.d.ts +4 -4
- package/dist/config/app.d.ts.map +1 -1
- package/dist/config/load.d.ts +2 -2
- package/dist/config/load.js +2 -2
- package/dist/config/resolve.d.ts +21 -6
- package/dist/config/resolve.d.ts.map +1 -1
- package/dist/config/resolve.js +165 -54
- package/dist/config/resolve.js.map +1 -1
- package/dist/engine/contract.d.ts +6 -1
- package/dist/engine/contract.d.ts.map +1 -1
- package/dist/engine/contract.js.map +1 -1
- package/dist/engine/index.d.ts +69 -6
- package/dist/engine/index.d.ts.map +1 -1
- package/dist/engine/index.js +25 -5
- package/dist/engine/index.js.map +1 -1
- package/dist/engine/session.d.ts.map +1 -1
- package/dist/engine/session.js +6 -0
- package/dist/engine/session.js.map +1 -1
- package/dist/engine/surface.d.ts +6 -2
- package/dist/engine/surface.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/internal/options.d.ts +9 -0
- package/dist/internal/options.d.ts.map +1 -0
- package/dist/internal/options.js +27 -0
- package/dist/internal/options.js.map +1 -0
- package/dist/locator/expression.js +1 -1
- package/dist/locator/expression.js.map +1 -1
- package/dist/locator/screen.d.ts.map +1 -1
- package/dist/locator/screen.js +4 -2
- package/dist/locator/screen.js.map +1 -1
- package/dist/report/build.d.ts +2 -1
- package/dist/report/build.d.ts.map +1 -1
- package/dist/report/build.js +2 -0
- package/dist/report/build.js.map +1 -1
- package/dist/report/json.d.ts +4 -0
- package/dist/report/json.d.ts.map +1 -0
- package/dist/report/json.js +8 -0
- package/dist/report/json.js.map +1 -0
- package/dist/report/junit.d.ts +7 -0
- package/dist/report/junit.d.ts.map +1 -1
- package/dist/report/junit.js +17 -0
- package/dist/report/junit.js.map +1 -1
- package/dist/report/list-model.d.ts +8 -5
- package/dist/report/list-model.d.ts.map +1 -1
- package/dist/report/list.d.ts +19 -4
- package/dist/report/list.d.ts.map +1 -1
- package/dist/report/list.js +67 -19
- package/dist/report/list.js.map +1 -1
- package/dist/report/running-tree.d.ts +5 -4
- package/dist/report/running-tree.d.ts.map +1 -1
- package/dist/report/running-tree.js +8 -7
- package/dist/report/running-tree.js.map +1 -1
- package/dist/run/artifacts.d.ts.map +1 -1
- package/dist/run/artifacts.js +20 -2
- package/dist/run/artifacts.js.map +1 -1
- package/dist/run/events.d.ts +14 -9
- package/dist/run/events.d.ts.map +1 -1
- package/dist/run/events.js +7 -7
- package/dist/run/events.js.map +1 -1
- package/dist/run/execute.d.ts +24 -5
- package/dist/run/execute.d.ts.map +1 -1
- package/dist/run/execute.js +67 -25
- package/dist/run/execute.js.map +1 -1
- package/dist/run/fixtures.d.ts +6 -1
- package/dist/run/fixtures.d.ts.map +1 -1
- package/dist/run/fixtures.js +0 -1
- package/dist/run/fixtures.js.map +1 -1
- package/dist/run/in-process.d.ts +1 -1
- package/dist/run/in-process.d.ts.map +1 -1
- package/dist/run/in-process.js +6 -3
- package/dist/run/in-process.js.map +1 -1
- package/dist/run/managed-process.d.ts.map +1 -1
- package/dist/run/managed-process.js +6 -2
- package/dist/run/managed-process.js.map +1 -1
- package/dist/run/records.d.ts +14 -1
- package/dist/run/records.d.ts.map +1 -1
- package/dist/run/runner.d.ts +17 -9
- package/dist/run/runner.d.ts.map +1 -1
- package/dist/run/runner.js +180 -101
- package/dist/run/runner.js.map +1 -1
- package/dist/run/scheduler.d.ts +4 -6
- package/dist/run/scheduler.d.ts.map +1 -1
- package/dist/run/scheduler.js +51 -23
- package/dist/run/scheduler.js.map +1 -1
- package/dist/run/serial.d.ts +2 -4
- package/dist/run/serial.d.ts.map +1 -1
- package/dist/run/serial.js +4 -2
- package/dist/run/serial.js.map +1 -1
- package/dist/run/steps.d.ts +25 -0
- package/dist/run/steps.d.ts.map +1 -1
- package/dist/run/steps.js +12 -0
- package/dist/run/steps.js.map +1 -1
- package/dist/run/unit-runner.d.ts +7 -2
- package/dist/run/unit-runner.d.ts.map +1 -1
- package/dist/run/units.d.ts +10 -0
- package/dist/run/units.d.ts.map +1 -1
- package/dist/run/units.js +11 -0
- package/dist/run/units.js.map +1 -1
- package/dist/run/worker/entry.js +1 -0
- package/dist/run/worker/entry.js.map +1 -1
- package/dist/run/worker/handle.d.ts +1 -1
- package/dist/run/worker/handle.d.ts.map +1 -1
- package/dist/run/worker/handle.js +1 -1
- package/dist/run/worker/handle.js.map +1 -1
- package/dist/run/worker/protocol.d.ts +2 -0
- package/dist/run/worker/protocol.d.ts.map +1 -1
- package/dist/run/worker/protocol.js.map +1 -1
- package/dist/run/worker/session.d.ts +2 -0
- package/dist/run/worker/session.d.ts.map +1 -1
- package/dist/run/worker/session.js +1 -0
- package/dist/run/worker/session.js.map +1 -1
- package/dist/telemetry/telemetry.d.ts +2 -3
- package/dist/telemetry/telemetry.d.ts.map +1 -1
- package/dist/telemetry/telemetry.js +2 -3
- package/dist/telemetry/telemetry.js.map +1 -1
- package/dist/types.d.ts +159 -37
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -6
- package/skills/e2e/SKILL.md +0 -2
- package/skills/e2e/references/agent.md +20 -12
- package/skills/e2e/references/debugging.md +3 -1
- package/skills/e2e/references/running.md +4 -3
- package/skills/e2e/references/setup.md +13 -10
- package/skills/e2e/references/writing-tests.md +6 -5
- package/dist/internal/trace.d.ts +0 -18
- package/dist/internal/trace.d.ts.map +0 -1
- package/dist/internal/trace.js +0 -46
- package/dist/internal/trace.js.map +0 -1
- package/dist/run/public.d.ts +0 -19
- package/dist/run/public.d.ts.map +0 -1
- package/dist/run/public.js +0 -11
- package/dist/run/public.js.map +0 -1
package/README.md
CHANGED
|
@@ -23,7 +23,7 @@ import type { E2EConfig } from '@e2edev/e2e';
|
|
|
23
23
|
import { playwright } from '@e2edev/playwright';
|
|
24
24
|
|
|
25
25
|
export default {
|
|
26
|
-
targets: [{ name: 'web',
|
|
26
|
+
targets: [{ name: 'web', engine: playwright({ url: process.env.APP_URL ?? 'http://localhost:3000' }) }],
|
|
27
27
|
} satisfies E2EConfig;
|
|
28
28
|
```
|
|
29
29
|
|
|
@@ -37,7 +37,6 @@ export default {
|
|
|
37
37
|
targets: [
|
|
38
38
|
{
|
|
39
39
|
name: 'web',
|
|
40
|
-
platform: 'web',
|
|
41
40
|
engine: playwright({
|
|
42
41
|
url: 'http://127.0.0.1:3000',
|
|
43
42
|
services: [
|
|
@@ -147,6 +146,12 @@ agent step, in the AI SDK devtools database shape. Open it with
|
|
|
147
146
|
[unbox-ai](https://github.com/tester-army/unbox-ai):
|
|
148
147
|
`npx unbox-ai .e2e/ai-trace.json`.
|
|
149
148
|
|
|
149
|
+
`e2e run --video` records every attempt: a WebM screencast of the page under
|
|
150
|
+
the attempt's artifact directory, named in the failure recap.
|
|
151
|
+
`artifacts: ['screenshot', 'trace', 'video']` turns it on for every run, and
|
|
152
|
+
`artifacts: { kinds: ['screenshot', 'trace', 'video'], video: { retain:
|
|
153
|
+
'on-failure' } }` keeps only the recordings of attempts that did not pass.
|
|
154
|
+
|
|
150
155
|
## Coding agents
|
|
151
156
|
|
|
152
157
|
`e2e init` installs a skill, `SKILL.md` plus one file per topic, into
|
|
@@ -163,7 +168,7 @@ npx --no-install e2e guide writing-tests # one topic
|
|
|
163
168
|
|
|
164
169
|
## Current limitations
|
|
165
170
|
|
|
166
|
-
- The HTML reporter
|
|
171
|
+
- The HTML reporter is not available.
|
|
167
172
|
- Reported steps carry no source locations.
|
|
168
173
|
|
|
169
174
|
## Contributing
|
|
@@ -3,15 +3,18 @@
|
|
|
3
3
|
* functions with no dispatch state: everything a step's arguments and an
|
|
4
4
|
* executor's verdict must satisfy before the harness trusts them.
|
|
5
5
|
*/
|
|
6
|
-
import type {
|
|
6
|
+
import type { ActOptions, AgentParams, JsonValue, Secret } from '../types.ts';
|
|
7
7
|
import { type StepVerdict } from './executor.ts';
|
|
8
8
|
/** Normalizes and bounds the instruction per spec 02. */
|
|
9
9
|
export declare function validateInstruction(instruction: string, api: string): string;
|
|
10
10
|
/**
|
|
11
|
-
* The `act` type has no `schema` or `vision
|
|
12
|
-
* checker who passes one still fails
|
|
11
|
+
* The `act` type has no `schema` or `vision`, and no key outside
|
|
12
|
+
* `ActOptions`; a caller outside the type checker who passes one still fails
|
|
13
|
+
* loudly instead of being silently ignored. The pre-0.8 shapes land here too:
|
|
14
|
+
* `act(instruction, params)` arrives as an options bag of the caller's own
|
|
15
|
+
* keys, and `act(instruction, params, options)` as a third argument.
|
|
13
16
|
*/
|
|
14
|
-
export declare function
|
|
17
|
+
export declare function validateActOptions(options: ActOptions | undefined, extraArguments: number): void;
|
|
15
18
|
/**
|
|
16
19
|
* Validates parameters and returns an inert, secret-free snapshot plus the
|
|
17
20
|
* declared secrets. A `Secret` value is projected to
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"act-validation.d.ts","sourceRoot":"","sources":["../../src/agent/act-validation.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"act-validation.d.ts","sourceRoot":"","sources":["../../src/agent/act-validation.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EAAE,UAAU,EAAkB,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE9F,OAAO,EAAmB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAWlE,yDAAyD;AACzD,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAa5E;AAID;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,SAAS,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,CA4BhG;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG;IAC/D,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,GAAG,SAAS,CAAC;IAC3D,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC,CAoBA;AAqBD,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,GAAG,WAAW,CAqCnF"}
|
|
@@ -26,13 +26,23 @@ export function validateInstruction(instruction, api) {
|
|
|
26
26
|
}
|
|
27
27
|
return normalized;
|
|
28
28
|
}
|
|
29
|
+
const ACT_OPTION_KEYS = new Set(['params', 'timeout', 'maxSteps', 'maxModelCalls']);
|
|
29
30
|
/**
|
|
30
|
-
* The `act` type has no `schema` or `vision
|
|
31
|
-
* checker who passes one still fails
|
|
31
|
+
* The `act` type has no `schema` or `vision`, and no key outside
|
|
32
|
+
* `ActOptions`; a caller outside the type checker who passes one still fails
|
|
33
|
+
* loudly instead of being silently ignored. The pre-0.8 shapes land here too:
|
|
34
|
+
* `act(instruction, params)` arrives as an options bag of the caller's own
|
|
35
|
+
* keys, and `act(instruction, params, options)` as a third argument.
|
|
32
36
|
*/
|
|
33
|
-
export function
|
|
37
|
+
export function validateActOptions(options, extraArguments) {
|
|
38
|
+
if (extraArguments > 0) {
|
|
39
|
+
throw new TestError('INVALID_ARGUMENT', 'agent.act takes two arguments: act(instruction, { params, timeout, maxSteps, maxModelCalls })');
|
|
40
|
+
}
|
|
34
41
|
if (options === undefined)
|
|
35
42
|
return;
|
|
43
|
+
if (typeof options !== 'object' || options === null || Array.isArray(options)) {
|
|
44
|
+
throw new TestError('INVALID_ARGUMENT', 'agent.act options must be a plain object');
|
|
45
|
+
}
|
|
36
46
|
const loose = options;
|
|
37
47
|
const unsupported = (name) => {
|
|
38
48
|
throw new ConfigurationError('UNSUPPORTED_CAPABILITY', `agent.act ${name} is not part of this milestone`);
|
|
@@ -41,6 +51,11 @@ export function rejectUnsupportedActOptions(options) {
|
|
|
41
51
|
unsupported('structured output (options.schema)');
|
|
42
52
|
if (loose.vision !== undefined)
|
|
43
53
|
unsupported('vision evidence (options.vision)');
|
|
54
|
+
const unknown = Object.keys(options).filter((key) => !ACT_OPTION_KEYS.has(key));
|
|
55
|
+
if (unknown.length > 0) {
|
|
56
|
+
const listed = unknown.map((key) => JSON.stringify(key)).join(', ');
|
|
57
|
+
throw new TestError('INVALID_ARGUMENT', `agent.act options has no ${unknown.length === 1 ? 'key' : 'keys'} ${listed}; the values an instruction refers to go under params: act(instruction, { params: { ${unknown.join(', ')} } })`);
|
|
58
|
+
}
|
|
44
59
|
}
|
|
45
60
|
/**
|
|
46
61
|
* Validates parameters and returns an inert, secret-free snapshot plus the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"act-validation.js","sourceRoot":"","sources":["../../src/agent/act-validation.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAoB,MAAM,eAAe,CAAC;AAElE,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAEhC,iEAAiE;AACjE,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAEpC,gFAAgF;AAChF,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,yDAAyD;AACzD,MAAM,UAAU,mBAAmB,CAAC,WAAmB,EAAE,GAAW;IAClE,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACjE,MAAM,IAAI,SAAS,CAAC,kBAAkB,EAAE,GAAG,GAAG,mCAAmC,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,UAAU,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC;IAC9D,IAAI,KAAK,GAAG,qBAAqB,EAAE,CAAC;QAClC,MAAM,IAAI,SAAS,CACjB,kBAAkB,EAClB,GAAG,GAAG,mBAAmB,KAAK,0BAA0B,qBAAqB,EAAE,CAChF,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"act-validation.js","sourceRoot":"","sources":["../../src/agent/act-validation.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAoB,MAAM,eAAe,CAAC;AAElE,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAEhC,iEAAiE;AACjE,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAEpC,gFAAgF;AAChF,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,yDAAyD;AACzD,MAAM,UAAU,mBAAmB,CAAC,WAAmB,EAAE,GAAW;IAClE,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACjE,MAAM,IAAI,SAAS,CAAC,kBAAkB,EAAE,GAAG,GAAG,mCAAmC,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,UAAU,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC;IAC9D,IAAI,KAAK,GAAG,qBAAqB,EAAE,CAAC;QAClC,MAAM,IAAI,SAAS,CACjB,kBAAkB,EAClB,GAAG,GAAG,mBAAmB,KAAK,0BAA0B,qBAAqB,EAAE,CAChF,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC,CAAC;AAEzG;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA+B,EAAE,cAAsB;IACxF,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,SAAS,CACjB,kBAAkB,EAClB,+FAA+F,CAChG,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO;IAClC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9E,MAAM,IAAI,SAAS,CAAC,kBAAkB,EAAE,0CAA0C,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,KAAK,GAAG,OAAmE,CAAC;IAClF,MAAM,WAAW,GAAG,CAAC,IAAY,EAAS,EAAE;QAC1C,MAAM,IAAI,kBAAkB,CAC1B,wBAAwB,EACxB,aAAa,IAAI,gCAAgC,CAClD,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;QAAE,WAAW,CAAC,oCAAoC,CAAC,CAAC;IAClF,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;QAAE,WAAW,CAAC,kCAAkC,CAAC,CAAC;IAChF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAChF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,MAAM,IAAI,SAAS,CACjB,kBAAkB,EAClB,4BAA4B,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,MAAM,uFAAuF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5L,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,MAA+B;IAI5D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IAC9E,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,SAAS,CAAC,kBAAkB,EAAE,yCAAyC,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;IAChE,iBAAiB,CAAC,YAAY,EAAE,kBAAkB,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC;IAC7D,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;QAC7B,MAAM,IAAI,SAAS,CACjB,kBAAkB,EAClB,wBAAwB,KAAK,oCAAoC,gBAAgB,EAAE,CACpF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAwC;QACvE,OAAO;KACR,CAAC;AACJ,CAAC;AAED,iFAAiF;AACjF,SAAS,cAAc,CAAC,KAAc,EAAE,OAA4B,EAAE,IAAkB;IACtF,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;IACtE,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,SAAS,CAAC,kBAAkB,EAAE,mCAAmC,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CACzF,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,OAAgB,EAAE,YAAoB;IACpE,MAAM,OAAO,GAAG,CAAC,KAAa,EAAS,EAAE;QACvC,MAAM,IAAI,UAAU,CAClB,sBAAsB,EACtB,aAAa,YAAY,kCAAkC,KAAK,EAAE,CACnE,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;IACrF,MAAM,SAAS,GAAG,OAAkC,CAAC;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvE,OAAO,OAAO,CAAC,kDAAkD,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACzD,OAAO,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;IACzC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,SAAS,IAAI,gBAAgB,CAAC,EAAE,CAAC;YACtE,OAAO,OAAO,CAAC,qBAAqB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,SAAuC,CAAC;IACrD,IAAI,MAAM,KAAK,QAAQ,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,4CAA4C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/E,OAAO,OAAO,CACZ,kDAAkD,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACrF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,MAAM;QACN,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC;QACnD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnD,CAAC;AACJ,CAAC"}
|
package/dist/agent/act.d.ts
CHANGED
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
* locate/judgment tier runs on. Trace-cache participation — replay, live
|
|
10
10
|
* recording, staging — lives beside the dispatch in `StepTraceSession`.
|
|
11
11
|
*/
|
|
12
|
-
import type {
|
|
12
|
+
import type { ActOptions, ActResult } from '../types.ts';
|
|
13
13
|
import type { AgentContext } from './invocation.ts';
|
|
14
14
|
/** Runs one `agent.act()` call as a harness-dispatched executor step. */
|
|
15
|
-
export declare function runActStep(runtime: AgentContext, instruction: string,
|
|
15
|
+
export declare function runActStep(runtime: AgentContext, instruction: string, options: ActOptions | undefined, extraArguments?: number): Promise<ActResult>;
|
|
16
16
|
/**
|
|
17
17
|
* Runs one `agent.assert()` call through the executor socket. Used when a
|
|
18
18
|
* custom executor is configured: the brain that plans flows also judges
|
package/dist/agent/act.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"act.d.ts","sourceRoot":"","sources":["../../src/agent/act.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAUH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"act.d.ts","sourceRoot":"","sources":["../../src/agent/act.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAUH,OAAO,KAAK,EACV,UAAU,EACV,SAAS,EAOV,MAAM,aAAa,CAAC;AAmBrB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAkFpD,yEAAyE;AACzE,wBAAsB,UAAU,CAC9B,OAAO,EAAE,YAAY,EACrB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,UAAU,GAAG,SAAS,EAC/B,cAAc,SAAI,GACjB,OAAO,CAAC,SAAS,CAAC,CAepB;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,SAAS,GAChF,OAAO,CAAC,IAAI,CAAC,CAqBf"}
|
package/dist/agent/act.js
CHANGED
|
@@ -11,30 +11,71 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { writeFileSync } from 'node:fs';
|
|
13
13
|
import { join } from 'node:path';
|
|
14
|
-
import {
|
|
15
|
-
import { ConfigurationError, TestError } from '../internal/errors.js';
|
|
14
|
+
import { asEngineError, ConfigurationError, TestError } from '../internal/errors.js';
|
|
16
15
|
import { timestamp } from '../internal/ids.js';
|
|
17
16
|
import { resolveNavigationUrl } from '../internal/urls.js';
|
|
18
17
|
import { AgentError, CATEGORY_BY_CODE, isAgentError, toAgentError } from './error.js';
|
|
19
|
-
import {
|
|
18
|
+
import { validateActOptions, validateInstruction, validateParams, validateVerdict, } from './act-validation.js';
|
|
20
19
|
import { resolveBoundedBudget, resolveTimeout } from './call-options.js';
|
|
21
20
|
import { RUNTIME_CODES, } from './executor.js';
|
|
22
21
|
import { isDerivedValue } from './derived.js';
|
|
23
22
|
import { projectPriorSteps, serializeLedger } from './ledger.js';
|
|
24
23
|
import { instantiateLanguageModel } from './model/sdk.js';
|
|
25
|
-
import { observationShape, pixelsForModel, prepareObservation, projectTree, settleObservation, } from './observation.js';
|
|
24
|
+
import { isTransitionalObservation, observationShape, pixelsForModel, prepareObservation, projectTree, settleObservation, } from './observation.js';
|
|
25
|
+
import { observationByteBudget } from './observation-budget.js';
|
|
26
26
|
import { boundedOperation, checkStepClock, instrumentPhase, recordPolicyEvent, retryingObserve } from './phases.js';
|
|
27
|
-
import { containerKey, describeAction } from './actions.js';
|
|
28
|
-
import { describePosition } from '../cache/relocate.js';
|
|
27
|
+
import { containerKey, describeAction, describeTarget } from './actions.js';
|
|
28
|
+
import { describePosition, relocateDescriptor } from '../cache/relocate.js';
|
|
29
29
|
import { authorizeSecretFill } from './secrets.js';
|
|
30
30
|
import { ModelUsage, tokenFields } from './usage.js';
|
|
31
31
|
import { StepTraceSession } from './step-cache.js';
|
|
32
|
+
/**
|
|
33
|
+
* Ceiling on one targeted grammar action: the time the engine may wait for a
|
|
34
|
+
* node to become actionable before the failure goes back to the model.
|
|
35
|
+
* `actionTimeout` bounds every engine operation and doubles as the judgment
|
|
36
|
+
* tier's clock, so projects raise it for slow models; a tap under a consent
|
|
37
|
+
* overlay then sits in the engine's actionability retry for the whole budget
|
|
38
|
+
* (90 s in the testbed) before the model learns anything, when the useful
|
|
39
|
+
* answer — what is in the way — is known within seconds. Navigation keeps the
|
|
40
|
+
* full budget: a slow page really can take that long to load.
|
|
41
|
+
*/
|
|
42
|
+
const MAX_TARGETED_ACTION_MS = 15_000;
|
|
43
|
+
/**
|
|
44
|
+
* How many times a targeted action re-finds its node after the engine reports
|
|
45
|
+
* it stale. One relocation covers a re-render between the observation and the
|
|
46
|
+
* action; the second covers a render tick landing between the fresh look and
|
|
47
|
+
* the action itself. A control that keeps vanishing faster than that is
|
|
48
|
+
* reported to the model as gone.
|
|
49
|
+
*/
|
|
50
|
+
const MAX_STALE_RELOCATIONS = 2;
|
|
51
|
+
/**
|
|
52
|
+
* The change wait after a scroll or a mutating project tool. A scroll moves
|
|
53
|
+
* nothing the tree records and a tool usually changes state the screen shows
|
|
54
|
+
* only after a reload, so most of these change no shape at all and a long
|
|
55
|
+
* wait is pure cost; a windowed list rendering its next rows, or a tool the
|
|
56
|
+
* page reacts to, does so within a few hundred milliseconds.
|
|
57
|
+
*/
|
|
58
|
+
const BRIEF_CHANGE_WAIT_MS = 500;
|
|
59
|
+
/**
|
|
60
|
+
* Observations kept for resolving an id the newest one no longer carries.
|
|
61
|
+
* A turn that batches actions addresses the screen it saw, while every
|
|
62
|
+
* action's own look re-observes; on an engine that mints ids per observation
|
|
63
|
+
* (a device), each look renumbers the tree. A few looks back is as far as one
|
|
64
|
+
* turn can reach.
|
|
65
|
+
*/
|
|
66
|
+
const MAX_RECENT_OBSERVATIONS = 8;
|
|
67
|
+
/**
|
|
68
|
+
* How far one grammar scroll moves. The engine's default flick is half the
|
|
69
|
+
* scrolled box; a deliberate scroll of three quarters keeps every row on
|
|
70
|
+
* screen at least once while covering a long feed in fewer actions.
|
|
71
|
+
*/
|
|
72
|
+
const SCROLL_MOMENTUM = 'slow';
|
|
32
73
|
/** Runs one `agent.act()` call as a harness-dispatched executor step. */
|
|
33
|
-
export async function runActStep(runtime, instruction,
|
|
74
|
+
export async function runActStep(runtime, instruction, options, extraArguments = 0) {
|
|
34
75
|
const normalized = validateInstruction(instruction, 'agent.act');
|
|
35
|
-
|
|
36
|
-
const { projected, secrets } = validateParams(params);
|
|
37
|
-
|
|
76
|
+
validateActOptions(options, extraArguments);
|
|
77
|
+
const { projected, secrets } = validateParams(options?.params);
|
|
78
|
+
return dispatchAgentStep(runtime, {
|
|
38
79
|
api: 'agent.act',
|
|
39
80
|
kind: 'act',
|
|
40
81
|
instruction: normalized,
|
|
@@ -45,7 +86,6 @@ export async function runActStep(runtime, instruction, params, options) {
|
|
|
45
86
|
maxSteps: options?.maxSteps,
|
|
46
87
|
maxModelCalls: options?.maxModelCalls,
|
|
47
88
|
});
|
|
48
|
-
return { ok: true };
|
|
49
89
|
}
|
|
50
90
|
/**
|
|
51
91
|
* Runs one `agent.assert()` call through the executor socket. Used when a
|
|
@@ -83,7 +123,7 @@ export async function runAssertStep(runtime, assertion, options) {
|
|
|
83
123
|
* (cache/context.ts).
|
|
84
124
|
*/
|
|
85
125
|
async function dispatchAgentStep(runtime, spec) {
|
|
86
|
-
|
|
126
|
+
return runtime.steps.run('agent', spec.api, spec.instruction, async () => {
|
|
87
127
|
const dispatch = new ActDispatch(runtime, spec);
|
|
88
128
|
try {
|
|
89
129
|
let verdict;
|
|
@@ -95,6 +135,7 @@ async function dispatchAgentStep(runtime, spec) {
|
|
|
95
135
|
}
|
|
96
136
|
dispatch.settle(validateVerdict(verdict, runtime.executor.name));
|
|
97
137
|
await dispatch.conclude('passed');
|
|
138
|
+
return dispatch.result();
|
|
98
139
|
}
|
|
99
140
|
catch (cause) {
|
|
100
141
|
await dispatch.conclude(isAgentError(cause) && cause.code === 'CANCELLED' ? 'cancelled' : 'failed');
|
|
@@ -124,6 +165,8 @@ class ActDispatch {
|
|
|
124
165
|
ledgerBytes: 0,
|
|
125
166
|
};
|
|
126
167
|
latest;
|
|
168
|
+
/** The newest observations of the step, oldest first; see MAX_RECENT_OBSERVATIONS. */
|
|
169
|
+
recent = [];
|
|
127
170
|
explanation;
|
|
128
171
|
/** First budget/timeout/cancel failure; runtime truth outranks the verdict. */
|
|
129
172
|
hardStop;
|
|
@@ -165,6 +208,17 @@ class ActDispatch {
|
|
|
165
208
|
stepIndex;
|
|
166
209
|
/** The last pixel decision recorded on this step: `allowed`, or the withheld reason. */
|
|
167
210
|
pixelsDecided;
|
|
211
|
+
/**
|
|
212
|
+
* The screen shape the newest committed action was resolved against, kept
|
|
213
|
+
* until the next settled observation has waited for the screen to leave it.
|
|
214
|
+
* Armed by actions whose effect shows in the tree; a secret fill leaves no
|
|
215
|
+
* visible trace and a scroll moves nothing the tree records, so neither
|
|
216
|
+
* arms it. Without this, the observation after a tap on a link reads the
|
|
217
|
+
* old page, stable and wrong, and the model repairs what already worked.
|
|
218
|
+
*/
|
|
219
|
+
pendingChange;
|
|
220
|
+
/** The change wait the pending action asked for; undefined takes the default. */
|
|
221
|
+
pendingChangeWaitMs;
|
|
168
222
|
constructor(runtime, spec) {
|
|
169
223
|
this.runtime = runtime;
|
|
170
224
|
this.spec = spec;
|
|
@@ -264,7 +318,7 @@ class ActDispatch {
|
|
|
264
318
|
buildActions() {
|
|
265
319
|
return {
|
|
266
320
|
tap: (target) => this.commitTargeted('tap', target, async (node) => {
|
|
267
|
-
await this.session.perform(node.ref, { kind: 'tap' }, this.
|
|
321
|
+
await this.session.perform(node.ref, { kind: 'tap' }, this.actionOperation());
|
|
268
322
|
return { name: 'tap', node };
|
|
269
323
|
}),
|
|
270
324
|
type: (target, value) => {
|
|
@@ -272,7 +326,7 @@ class ActDispatch {
|
|
|
272
326
|
throw new TestError('INVALID_ARGUMENT', 'type value must be a string');
|
|
273
327
|
}
|
|
274
328
|
return this.commitTargeted('type', target, async (node) => {
|
|
275
|
-
await this.session.perform(node.ref, { kind: 'fill', value, sensitive: false }, this.
|
|
329
|
+
await this.session.perform(node.ref, { kind: 'fill', value, sensitive: false }, this.actionOperation());
|
|
276
330
|
return { name: 'type', node, value };
|
|
277
331
|
});
|
|
278
332
|
},
|
|
@@ -282,7 +336,7 @@ class ActDispatch {
|
|
|
282
336
|
throw new TestError('INVALID_ARGUMENT', 'press key must be a short non-empty string');
|
|
283
337
|
}
|
|
284
338
|
return this.commitTargeted('press', target, async (node) => {
|
|
285
|
-
await this.session.perform(node.ref, { kind: 'press', key }, this.
|
|
339
|
+
await this.session.perform(node.ref, { kind: 'press', key }, this.actionOperation());
|
|
286
340
|
return { name: 'press', node, key };
|
|
287
341
|
});
|
|
288
342
|
},
|
|
@@ -291,7 +345,7 @@ class ActDispatch {
|
|
|
291
345
|
throw new TestError('INVALID_ARGUMENT', 'select value must be a non-empty option label');
|
|
292
346
|
}
|
|
293
347
|
return this.commitTargeted('selectOption', target, async (node) => {
|
|
294
|
-
await this.session.perform(node.ref, { kind: 'selectOption', value }, this.
|
|
348
|
+
await this.session.perform(node.ref, { kind: 'selectOption', value }, this.actionOperation());
|
|
295
349
|
return { name: 'select', node, value };
|
|
296
350
|
});
|
|
297
351
|
},
|
|
@@ -344,6 +398,7 @@ class ActDispatch {
|
|
|
344
398
|
redact: this.redact,
|
|
345
399
|
testIdAttribute: this.runtime.config.testIdAttribute,
|
|
346
400
|
currentPath: (nodes) => this.currentPath(nodes !== undefined && nodes === this.latest?.nodes ? this.latest : undefined),
|
|
401
|
+
replaying: (active) => this.runtime.steps.replaying(active),
|
|
347
402
|
};
|
|
348
403
|
}
|
|
349
404
|
/** Best-effort current location path + query, for trace preconditions. */
|
|
@@ -468,7 +523,7 @@ class ActDispatch {
|
|
|
468
523
|
this.modelId = usage.modelId;
|
|
469
524
|
this.runtime.steps.recordEvent({
|
|
470
525
|
kind: 'model',
|
|
471
|
-
startedAt: timestamp(),
|
|
526
|
+
startedAt: usage?.startedAt ?? timestamp(),
|
|
472
527
|
durationMs: Math.max(0, Math.round(usage?.durationMs ?? 0)),
|
|
473
528
|
status: 'passed',
|
|
474
529
|
name: 'executor',
|
|
@@ -501,10 +556,25 @@ class ActDispatch {
|
|
|
501
556
|
this.reserveAction();
|
|
502
557
|
this.stepCache?.recordGap(call.name);
|
|
503
558
|
}
|
|
504
|
-
|
|
559
|
+
const value = await instrumentPhase(this.runtime, { api: this.spec.api, kind: 'engine', phase: 'agent.action', name: `tool:${call.name}` }, body);
|
|
560
|
+
// A tool the page reacts to at once is read after the reaction; one
|
|
561
|
+
// whose effect shows only after a reload costs the brief wait, not two seconds.
|
|
562
|
+
if (call.mutates)
|
|
563
|
+
this.armChange(BRIEF_CHANGE_WAIT_MS);
|
|
564
|
+
return value;
|
|
505
565
|
};
|
|
506
566
|
return call.mutates ? this.serialized(run) : run();
|
|
507
567
|
}
|
|
568
|
+
/** What the settled step did, for the caller of `agent.act()`. */
|
|
569
|
+
result() {
|
|
570
|
+
const cache = this.stepCache?.cacheInfo;
|
|
571
|
+
return {
|
|
572
|
+
summary: this.explanation ?? '',
|
|
573
|
+
...(cache === undefined ? {} : { cache }),
|
|
574
|
+
modelCalls: this.metrics.modelCalls,
|
|
575
|
+
actions: this.metrics.actionSteps,
|
|
576
|
+
};
|
|
577
|
+
}
|
|
508
578
|
/** Attaches metrics, model provenance, and the verdict explanation to the step. */
|
|
509
579
|
finish() {
|
|
510
580
|
this.closed = true;
|
|
@@ -556,6 +626,10 @@ class ActDispatch {
|
|
|
556
626
|
operation() {
|
|
557
627
|
return boundedOperation(this.runtime.engine, this.runtime.config.actionTimeout, this.deadline);
|
|
558
628
|
}
|
|
629
|
+
/** The operation context of one targeted action; see MAX_TARGETED_ACTION_MS. */
|
|
630
|
+
actionOperation() {
|
|
631
|
+
return boundedOperation(this.runtime.engine, Math.min(this.runtime.config.actionTimeout, MAX_TARGETED_ACTION_MS), this.deadline);
|
|
632
|
+
}
|
|
559
633
|
/** Resolves the configured model once; executors that never read it never pay. */
|
|
560
634
|
resolveModel() {
|
|
561
635
|
if (!this.sdkModelResolved) {
|
|
@@ -652,6 +726,14 @@ class ActDispatch {
|
|
|
652
726
|
// A tainted viewport never captures pixels: the engine would mask what it
|
|
653
727
|
// knows about, and the secret may be anywhere on screen by now.
|
|
654
728
|
const capturePixels = pixels && !this.runtime.taint.value;
|
|
729
|
+
// A settled look consumes the pending change: it waits for the screen to
|
|
730
|
+
// leave the pre-action shape once, and later looks read the screen as is.
|
|
731
|
+
const changedFrom = settle ? this.pendingChange : undefined;
|
|
732
|
+
const changeWaitMs = settle ? this.pendingChangeWaitMs : undefined;
|
|
733
|
+
if (settle) {
|
|
734
|
+
this.pendingChange = undefined;
|
|
735
|
+
this.pendingChangeWaitMs = undefined;
|
|
736
|
+
}
|
|
655
737
|
const observation = await instrumentPhase(this.runtime, { api: this.spec.api, kind: 'observation', phase: 'agent.observe' }, () => settle
|
|
656
738
|
? settleObservation(() => this.captureObservation(capturePixels), observationShape, {
|
|
657
739
|
remainingMs: () => this.deadline.remaining(),
|
|
@@ -659,12 +741,30 @@ class ActDispatch {
|
|
|
659
741
|
// the attempt signal alone would let settling outlive the step
|
|
660
742
|
// by one poll interval.
|
|
661
743
|
signal: this.stepSignal,
|
|
662
|
-
})
|
|
744
|
+
}, { changedFrom, changeWaitMs, transitional: isTransitionalObservation })
|
|
663
745
|
: this.captureObservation(capturePixels), (prepared) => ({ count: prepared.nodes.size, bytes: prepared.bytes }));
|
|
664
|
-
this.
|
|
746
|
+
this.publish(observation);
|
|
665
747
|
this.metrics.observationBytes = Math.max(this.metrics.observationBytes, observation.bytes);
|
|
666
748
|
return observation;
|
|
667
749
|
}
|
|
750
|
+
/** Makes an observation the newest and remembers it among the recent ones. */
|
|
751
|
+
publish(observation) {
|
|
752
|
+
this.latest = observation;
|
|
753
|
+
this.recent.push(observation);
|
|
754
|
+
if (this.recent.length > MAX_RECENT_OBSERVATIONS)
|
|
755
|
+
this.recent.shift();
|
|
756
|
+
}
|
|
757
|
+
/**
|
|
758
|
+
* Arms the change wait: the next settled observation waits for the screen
|
|
759
|
+
* to leave the newest observation's shape, for the default two seconds or
|
|
760
|
+
* the given brief window.
|
|
761
|
+
*/
|
|
762
|
+
armChange(waitMs) {
|
|
763
|
+
if (this.latest === undefined)
|
|
764
|
+
return;
|
|
765
|
+
this.pendingChange = observationShape(this.latest);
|
|
766
|
+
this.pendingChangeWaitMs = waitMs;
|
|
767
|
+
}
|
|
668
768
|
/** One raw observation capture: retried at the engine, then redacted and bounded. */
|
|
669
769
|
async captureObservation(pixels) {
|
|
670
770
|
const raw = await retryingObserve({
|
|
@@ -676,11 +776,31 @@ class ActDispatch {
|
|
|
676
776
|
});
|
|
677
777
|
return prepareObservation(raw, {
|
|
678
778
|
redact: this.runtime.redact,
|
|
679
|
-
maxBytes: this.
|
|
779
|
+
maxBytes: this.observationByteBudget(pixels),
|
|
680
780
|
testIdAttribute: this.runtime.config.testIdAttribute,
|
|
681
781
|
});
|
|
682
782
|
}
|
|
683
|
-
/**
|
|
783
|
+
/**
|
|
784
|
+
* Bytes one act observation may contribute to a model turn, clamped like a
|
|
785
|
+
* judgment's. The loop's history grows past what any one observation costs,
|
|
786
|
+
* so the fixed part counted here is what every turn resends: the project
|
|
787
|
+
* context and the prior-step ledger.
|
|
788
|
+
*/
|
|
789
|
+
observationByteBudget(pixels) {
|
|
790
|
+
const { config } = this.runtime;
|
|
791
|
+
return observationByteBudget({
|
|
792
|
+
maxObservationBytes: config.agent.maxObservationBytes,
|
|
793
|
+
maxModelTokensPerCall: config.limits.maxModelTokensPerCall,
|
|
794
|
+
}, { fixedBytes: this.metrics.contextBytes + this.metrics.ledgerBytes, pixels });
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* Resolves an executor target against the newest observation. An id the
|
|
798
|
+
* newest observation no longer carries, but a recent one did, is re-found
|
|
799
|
+
* in the newest one by its descriptor: a turn that batches actions keeps
|
|
800
|
+
* addressing the screen it saw while each action's own look re-observes,
|
|
801
|
+
* and an engine that mints ids per observation renumbers the tree between
|
|
802
|
+
* them. Exactly one match, or the id counts as gone.
|
|
803
|
+
*/
|
|
684
804
|
resolveTarget(target) {
|
|
685
805
|
if (typeof target?.id !== 'string' || target.id === '') {
|
|
686
806
|
throw new TestError('INVALID_ARGUMENT', 'action target must be { id: string }');
|
|
@@ -690,12 +810,29 @@ class ActDispatch {
|
|
|
690
810
|
if (latest === undefined) {
|
|
691
811
|
throw new AgentError('LOCATOR_NOT_FOUND', 'no observation has been captured yet; observe before acting');
|
|
692
812
|
}
|
|
693
|
-
const node = latest.nodes.get(id);
|
|
813
|
+
const node = latest.nodes.get(id) ?? this.refound(id, latest);
|
|
694
814
|
if (node === undefined) {
|
|
695
|
-
throw new AgentError('LOCATOR_NOT_FOUND', `node #${id} is not
|
|
815
|
+
throw new AgentError('LOCATOR_NOT_FOUND', `node #${id} is not on the current screen (observation ${latest.revision}); it was removed or never existed`);
|
|
696
816
|
}
|
|
697
817
|
return node;
|
|
698
818
|
}
|
|
819
|
+
/** The newest node matching the descriptor of what `id` named in a recent observation, if exactly one. */
|
|
820
|
+
refound(id, latest) {
|
|
821
|
+
let earlier;
|
|
822
|
+
for (let index = this.recent.length - 1; index >= 0 && earlier === undefined; index -= 1) {
|
|
823
|
+
earlier = this.recent[index].nodes.get(id);
|
|
824
|
+
}
|
|
825
|
+
if (earlier === undefined)
|
|
826
|
+
return undefined;
|
|
827
|
+
const descriptor = describeTarget(earlier, this.redact, this.runtime.config.testIdAttribute);
|
|
828
|
+
if (descriptor === undefined)
|
|
829
|
+
return undefined;
|
|
830
|
+
const relocated = relocateDescriptor(descriptor, latest.nodes, {
|
|
831
|
+
redact: this.redact,
|
|
832
|
+
testIdAttribute: this.runtime.config.testIdAttribute,
|
|
833
|
+
});
|
|
834
|
+
return relocated.kind === 'found' ? latest.nodes.get(relocated.id) : undefined;
|
|
835
|
+
}
|
|
699
836
|
/**
|
|
700
837
|
* Runs one grammar action against the action budget, recorded as an engine
|
|
701
838
|
* event. The body performs the engine call and returns the committed
|
|
@@ -718,6 +855,12 @@ class ActDispatch {
|
|
|
718
855
|
this.checkpoint(cause);
|
|
719
856
|
throw cause;
|
|
720
857
|
}
|
|
858
|
+
// The effect may still be arriving: the next settled observation waits for
|
|
859
|
+
// the screen to leave the shape this action was resolved against. A secret
|
|
860
|
+
// fill leaves no visible trace and arms nothing; a scroll waits briefly for
|
|
861
|
+
// rows a windowed or lazy list renders.
|
|
862
|
+
if (name !== 'typeSecret')
|
|
863
|
+
this.armChange(name === 'scroll' ? BRIEF_CHANGE_WAIT_MS : undefined);
|
|
721
864
|
if (this.stepCache === undefined)
|
|
722
865
|
return;
|
|
723
866
|
// A typed value the step derived at run time is this run's data, not the
|
|
@@ -729,51 +872,85 @@ class ActDispatch {
|
|
|
729
872
|
}
|
|
730
873
|
this.stepCache.record(action);
|
|
731
874
|
}
|
|
732
|
-
/**
|
|
875
|
+
/**
|
|
876
|
+
* One action against a resolved node. A node the engine reports stale is
|
|
877
|
+
* re-found by its descriptor against a fresh capture and the action retried
|
|
878
|
+
* (`MAX_STALE_RELOCATIONS` times): a list that remounts its rows between
|
|
879
|
+
* the observation and the action keeps the control on screen under a dead
|
|
880
|
+
* handle, and the control, not the handle, is what the model asked for. A
|
|
881
|
+
* descriptor that matches nothing or several nodes fails the action instead.
|
|
882
|
+
*/
|
|
733
883
|
commitTargeted(name, target, perform) {
|
|
734
884
|
return this.runAction(name, async () => {
|
|
735
|
-
|
|
736
|
-
//
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
885
|
+
let node = this.resolveTarget(target);
|
|
886
|
+
// resolveTarget guarantees an observation; it is the one the node came from.
|
|
887
|
+
let observation = this.latest;
|
|
888
|
+
for (let relocations = 0;; relocations += 1) {
|
|
889
|
+
// The container the node sits in is captured with it: that is what
|
|
890
|
+
// tells this row's "Delete" from the next row's when the flow replays.
|
|
891
|
+
const within = containerKey(node.ref.id, observation.nodes, observation.parents, this.redact);
|
|
892
|
+
// When the description still matches several controls, the position among
|
|
893
|
+
// them is recorded too; a replay that finds the same number picks the same one.
|
|
894
|
+
const position = describePosition(node, within, observation.nodes, {
|
|
745
895
|
redact: this.redact,
|
|
746
896
|
testIdAttribute: this.runtime.config.testIdAttribute,
|
|
747
897
|
});
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
898
|
+
try {
|
|
899
|
+
const action = await perform(node);
|
|
900
|
+
return {
|
|
901
|
+
...action,
|
|
902
|
+
...(within === undefined ? {} : { within }),
|
|
903
|
+
...(position === undefined ? {} : { position }),
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
catch (cause) {
|
|
907
|
+
if (asEngineError(cause)?.code !== 'NODE_STALE')
|
|
908
|
+
throw cause;
|
|
909
|
+
const relocated = relocations < MAX_STALE_RELOCATIONS ? await this.relocateStale(node) : undefined;
|
|
910
|
+
if (relocated === undefined) {
|
|
911
|
+
throw new AgentError('LOCATOR_NOT_FOUND', 'the target node left the screen before the action reached it', { cause });
|
|
912
|
+
}
|
|
913
|
+
node = relocated.node;
|
|
914
|
+
observation = relocated.observation;
|
|
759
915
|
}
|
|
760
|
-
throw cause;
|
|
761
916
|
}
|
|
762
917
|
});
|
|
763
918
|
}
|
|
919
|
+
/**
|
|
920
|
+
* Re-finds a node that went stale: one fresh capture, then the trace
|
|
921
|
+
* recorder's own descriptor matching (`cache/relocate.ts`) against it,
|
|
922
|
+
* exactly one match or nothing. The capture is taken directly rather than
|
|
923
|
+
* through the serialized observe: this runs inside a serialized action
|
|
924
|
+
* body, and a queued observation would wait on its own caller.
|
|
925
|
+
*/
|
|
926
|
+
async relocateStale(stale) {
|
|
927
|
+
const options = { redact: this.redact, testIdAttribute: this.runtime.config.testIdAttribute };
|
|
928
|
+
const descriptor = describeTarget(stale, this.redact, this.runtime.config.testIdAttribute);
|
|
929
|
+
if (descriptor === undefined)
|
|
930
|
+
return undefined;
|
|
931
|
+
this.checkpoint();
|
|
932
|
+
const observation = await instrumentPhase(this.runtime, { api: this.spec.api, kind: 'observation', phase: 'agent.observe', name: 'relocate' }, () => this.captureObservation(false), (prepared) => ({ count: prepared.nodes.size, bytes: prepared.bytes }));
|
|
933
|
+
this.publish(observation);
|
|
934
|
+
this.metrics.observationBytes = Math.max(this.metrics.observationBytes, observation.bytes);
|
|
935
|
+
const relocated = relocateDescriptor(descriptor, observation.nodes, options);
|
|
936
|
+
if (relocated.kind !== 'found')
|
|
937
|
+
return undefined;
|
|
938
|
+
const node = observation.nodes.get(relocated.id);
|
|
939
|
+
return node === undefined ? undefined : { node, observation };
|
|
940
|
+
}
|
|
764
941
|
async scroll(direction, target) {
|
|
765
942
|
if (!['up', 'down', 'left', 'right'].includes(direction)) {
|
|
766
943
|
throw new TestError('INVALID_ARGUMENT', `invalid scroll direction "${String(direction)}"`);
|
|
767
944
|
}
|
|
768
945
|
if (target === undefined) {
|
|
769
946
|
await this.runAction('scroll', async () => {
|
|
770
|
-
await this.session.swipe(direction,
|
|
947
|
+
await this.session.swipe(direction, SCROLL_MOMENTUM, this.actionOperation());
|
|
771
948
|
return { name: 'scroll', direction };
|
|
772
949
|
});
|
|
773
950
|
return;
|
|
774
951
|
}
|
|
775
952
|
await this.commitTargeted('scroll', target, async (node) => {
|
|
776
|
-
await this.session.perform(node.ref, { kind: 'swipe', direction }, this.
|
|
953
|
+
await this.session.perform(node.ref, { kind: 'swipe', direction, momentum: SCROLL_MOMENTUM }, this.actionOperation());
|
|
777
954
|
return { name: 'scroll', direction, node };
|
|
778
955
|
});
|
|
779
956
|
}
|
|
@@ -791,10 +968,10 @@ class ActDispatch {
|
|
|
791
968
|
await this.commitTargeted('typeSecret', target, async (node) => {
|
|
792
969
|
const plaintext = await authorizeSecretFill({
|
|
793
970
|
session: this.session,
|
|
794
|
-
operation: () => this.
|
|
971
|
+
operation: () => this.actionOperation(),
|
|
795
972
|
recordPolicy: (policy, decision, code) => this.recordPolicy(policy, decision, code),
|
|
796
973
|
}, this.runtime, secret, node);
|
|
797
|
-
await this.session.perform(node.ref, { kind: 'fill', value: plaintext, sensitive: true }, this.
|
|
974
|
+
await this.session.perform(node.ref, { kind: 'fill', value: plaintext, sensitive: true }, this.actionOperation());
|
|
798
975
|
this.runtime.taint.value = true;
|
|
799
976
|
// Recorded by stable name only; replay re-runs the full authorization.
|
|
800
977
|
return { name: 'typeSecret', node, secret: name };
|