e2e 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build.json +1 -1
- package/dist/agent/act.js +7 -4
- package/dist/agent/action-dispatcher.js +4 -5
- package/dist/agent/actions.d.ts +32 -23
- package/dist/agent/actions.js +38 -24
- package/dist/agent/model/sdk.js +2 -0
- package/dist/agent/observation-feed.d.ts +5 -6
- package/dist/agent/observation-feed.js +5 -7
- package/dist/agent/observation.d.ts +40 -11
- package/dist/agent/observation.js +109 -59
- package/dist/agent/point-tap.d.ts +8 -8
- package/dist/agent/point-tap.js +1 -1
- package/dist/agent/replay.d.ts +19 -8
- package/dist/agent/replay.js +11 -14
- package/dist/agent/scroll-target.d.ts +2 -3
- package/dist/agent/step-cache.d.ts +41 -14
- package/dist/agent/step-cache.js +147 -62
- package/dist/agent/tool-loop.js +2 -0
- package/dist/cache/anchors.d.ts +72 -23
- package/dist/cache/anchors.js +233 -60
- package/dist/cache/context.d.ts +25 -14
- package/dist/cache/context.js +50 -8
- package/dist/cache/decide.d.ts +2 -2
- package/dist/cache/decide.js +4 -6
- package/dist/cache/identity.d.ts +18 -4
- package/dist/cache/identity.js +9 -4
- package/dist/cache/recorder.d.ts +15 -1
- package/dist/cache/recorder.js +36 -23
- package/dist/cache/rekeyed.d.ts +46 -0
- package/dist/cache/rekeyed.js +89 -0
- package/dist/cache/relocate.d.ts +14 -14
- package/dist/cache/relocate.js +33 -25
- package/dist/cache/route.d.ts +32 -33
- package/dist/cache/route.js +94 -50
- package/dist/cache/store.d.ts +6 -0
- package/dist/cache/store.js +21 -1
- package/dist/cache/trace.d.ts +53 -14
- package/dist/cache/trace.js +70 -28
- package/dist/cli/index.js +8 -8
- package/dist/cli/init/engines.d.ts +1 -3
- package/dist/cli/init/engines.js +2 -2
- package/dist/cli/init/gateways.d.ts +1 -1
- package/dist/cli/init/gateways.js +11 -1
- package/dist/cli/init/sibling-versions.json +1 -1
- package/dist/cli/init/versions.d.ts +0 -8
- package/dist/cli/init/versions.js +1 -10
- package/dist/config/resolve.d.ts +1 -0
- package/dist/config/resolve.js +33 -1
- package/dist/engine/index.d.ts +2 -2
- package/dist/expect/async.js +2 -1
- package/dist/expect/poll-scope.d.ts +52 -0
- package/dist/expect/poll-scope.js +113 -0
- package/dist/expect/poll.js +31 -4
- package/dist/expect/values.js +11 -4
- package/dist/internal/abandoned.d.ts +11 -0
- package/dist/internal/abandoned.js +23 -0
- package/dist/internal/brands.d.ts +2 -0
- package/dist/internal/brands.js +2 -0
- package/dist/internal/client-identity.d.ts +9 -0
- package/dist/internal/client-identity.js +14 -0
- package/dist/internal/redact.d.ts +7 -4
- package/dist/internal/redact.js +116 -23
- package/dist/internal/time.d.ts +6 -0
- package/dist/internal/time.js +25 -13
- package/dist/internal/urls.d.ts +9 -5
- package/dist/internal/urls.js +19 -10
- package/dist/locator/engine.d.ts +12 -2
- package/dist/locator/engine.js +8 -3
- package/dist/locator/screen.js +28 -4
- package/dist/mcp/catalog.d.ts +3 -2
- package/dist/mcp/catalog.js +6 -6
- package/dist/mcp/session.js +1 -1
- package/dist/oauth/chatgpt.js +3 -23
- package/dist/oauth/cli.js +1 -0
- package/dist/oauth/copilot.d.ts +5 -4
- package/dist/oauth/copilot.js +80 -7
- package/dist/oauth/grok.js +2 -1
- package/dist/oauth/models.js +2 -1
- package/dist/oauth/opencode-console.d.ts +10 -0
- package/dist/oauth/opencode-console.js +133 -0
- package/dist/oauth/providers/github-copilot.d.ts +12 -0
- package/dist/oauth/providers/github-copilot.js +77 -23
- package/dist/oauth/providers/opencode-console.d.ts +31 -0
- package/dist/oauth/providers/opencode-console.js +148 -0
- package/dist/oauth/providers.d.ts +2 -6
- package/dist/oauth/providers.js +3 -7
- package/dist/oauth/responses.d.ts +9 -0
- package/dist/oauth/responses.js +23 -0
- package/dist/report/build.d.ts +26 -0
- package/dist/report/build.js +16 -8
- package/dist/report/failure-text.d.ts +6 -1
- package/dist/report/failure-text.js +14 -4
- package/dist/report/format.d.ts +10 -5
- package/dist/report/format.js +12 -5
- package/dist/report/junit.js +11 -12
- package/dist/report/list.d.ts +1 -0
- package/dist/report/list.js +38 -18
- package/dist/report/markdown.d.ts +1 -1
- package/dist/report/markdown.js +37 -14
- package/dist/report/repeats.d.ts +6 -1
- package/dist/report/repeats.js +12 -3
- package/dist/run/artifacts.d.ts +18 -0
- package/dist/run/artifacts.js +68 -1
- package/dist/run/execute.d.ts +9 -0
- package/dist/run/execute.js +49 -24
- package/dist/run/extended-fixtures.js +6 -2
- package/dist/run/failure-evidence.js +4 -2
- package/dist/run/fixtures.js +6 -8
- package/dist/run/in-process.d.ts +2 -0
- package/dist/run/in-process.js +1 -0
- package/dist/run/last-run.d.ts +42 -8
- package/dist/run/last-run.js +121 -14
- package/dist/run/realm.d.ts +4 -1
- package/dist/run/realm.js +20 -6
- package/dist/run/records.d.ts +16 -0
- package/dist/run/records.js +26 -0
- package/dist/run/retry.d.ts +3 -1
- package/dist/run/retry.js +4 -2
- package/dist/run/runner.d.ts +1 -1
- package/dist/run/runner.js +69 -30
- package/dist/run/serial.d.ts +2 -0
- package/dist/run/serial.js +8 -5
- package/dist/run/steps.d.ts +0 -6
- package/dist/run/steps.js +3 -20
- package/dist/run/worker/entry.js +5 -4
- package/dist/run/worker/protocol.d.ts +2 -0
- package/dist/run/worker/session.d.ts +2 -0
- package/dist/run/worker/session.js +1 -0
- package/dist/telemetry/events.js +1 -0
- package/dist/types.d.ts +9 -3
- package/docs/agents.mdx +6 -3
- package/docs/assertions.mdx +94 -0
- package/docs/authentication.mdx +3 -1
- package/docs/browser.mdx +42 -6
- package/docs/cache.mdx +64 -18
- package/docs/ci.mdx +11 -7
- package/docs/debugging.mdx +6 -3
- package/docs/executors.mdx +4 -2
- package/docs/github.mdx +12 -4
- package/docs/{writing-tests.mdx → goals.mdx} +21 -101
- package/docs/index.mdx +4 -4
- package/docs/integrations/kernel.mdx +1 -1
- package/docs/locators.mdx +64 -0
- package/docs/migrate/cypress.mdx +2 -2
- package/docs/migrate/maestro.mdx +2 -2
- package/docs/migrate/playwright.mdx +19 -16
- package/docs/migrate/selenium.mdx +2 -2
- package/docs/mobile.mdx +21 -4
- package/docs/models.mdx +53 -7
- package/docs/physical-devices.mdx +76 -0
- package/docs/quickstart.mdx +4 -3
- package/docs/reference/app.mdx +1 -1
- package/docs/reference/cli.mdx +30 -13
- package/docs/reference/config.mdx +25 -7
- package/docs/reference/engine.mdx +3 -3
- package/docs/reference/environment.mdx +8 -1
- package/docs/reference/errors.mdx +5 -5
- package/docs/reference/expect.mdx +34 -12
- package/docs/reference/mobile.mdx +23 -13
- package/docs/reference/reporters.mdx +8 -2
- package/docs/reference/screen.mdx +30 -7
- package/docs/reference/test.mdx +11 -4
- package/docs/reference/web.mdx +125 -21
- package/docs/security.mdx +26 -16
- package/docs/subscriptions.mdx +91 -12
- package/docs/telemetry.mdx +1 -1
- package/package.json +16 -3
- package/schema/report-v1.schema.json +14 -1
- package/skills/e2e/SKILL.md +2 -2
- package/skills/e2e/references/agent.md +22 -11
- package/skills/e2e/references/debugging.md +4 -4
- package/skills/e2e/references/running.md +13 -8
- package/skills/e2e/references/setup.md +36 -15
- package/skills/e2e/references/writing-tests.md +34 -17
|
@@ -15,10 +15,11 @@ const UNAVAILABLE_TREE_MARKER = '[semantic capture unavailable: no nodes were re
|
|
|
15
15
|
*
|
|
16
16
|
* Pixels whose masking the engine cannot prove (fewer masked regions than
|
|
17
17
|
* secure nodes) are withheld before they reach a model or disk; the semantic
|
|
18
|
-
* tree never carries secure values.
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* tree never carries secure values. Every string of every node is redacted
|
|
19
|
+
* once, here (`redactNode`): registered secret values are replaced by their
|
|
20
|
+
* stable secret name, and so is the leading part of one a field the engine
|
|
21
|
+
* cut at its limit ends with. A field holding one keeps no selection, as a
|
|
22
|
+
* secure field keeps none.
|
|
22
23
|
*/
|
|
23
24
|
export function prepareObservation(observation, options) {
|
|
24
25
|
const location = observation.location === undefined ? undefined : options.redact(observation.location);
|
|
@@ -42,11 +43,10 @@ export function prepareObservation(observation, options) {
|
|
|
42
43
|
pixels: pixels.cleared,
|
|
43
44
|
};
|
|
44
45
|
}
|
|
45
|
-
const tree =
|
|
46
|
+
const tree = redactNode(observation.tree, options);
|
|
46
47
|
const nodes = new Map();
|
|
47
48
|
const parents = new Map();
|
|
48
49
|
indexNodes(tree, nodes, parents);
|
|
49
|
-
const redact = options.redact;
|
|
50
50
|
const lines = [];
|
|
51
51
|
const encoder = new TextEncoder();
|
|
52
52
|
// The marker is reserved up front so a truncated observation still fits the
|
|
@@ -63,7 +63,7 @@ export function prepareObservation(observation, options) {
|
|
|
63
63
|
const emit = (node, depth) => {
|
|
64
64
|
if (cutByBudget)
|
|
65
65
|
return;
|
|
66
|
-
const line = formatNode(node, depth,
|
|
66
|
+
const line = formatNode(node, depth, options.appOrigin);
|
|
67
67
|
const size = encoder.encode(`${line}\n`).byteLength;
|
|
68
68
|
if (lines.length > 0 && bytes + size > budget) {
|
|
69
69
|
cutByBudget = true;
|
|
@@ -97,40 +97,94 @@ export function prepareObservation(observation, options) {
|
|
|
97
97
|
...(pixels.withheld === undefined ? {} : { pixelsWithheld: pixels.withheld }),
|
|
98
98
|
};
|
|
99
99
|
}
|
|
100
|
-
/**
|
|
101
|
-
const
|
|
102
|
-
['name', OBSERVED_NAME_LIMIT],
|
|
103
|
-
['text', OBSERVED_TEXT_LIMIT],
|
|
104
|
-
['value', OBSERVED_TEXT_LIMIT],
|
|
105
|
-
['selection', OBSERVED_TEXT_LIMIT],
|
|
106
|
-
];
|
|
100
|
+
/** Leaves a field the engine fills from a closed set (an enum, booleans, numbers) or mints itself as it is. */
|
|
101
|
+
const keep = (value) => value;
|
|
107
102
|
/**
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
* with nothing changed is returned as it is.
|
|
103
|
+
* How each field of a node is redacted, one entry per field of
|
|
104
|
+
* `SemanticNode`: a field added to the contract does not compile until it
|
|
105
|
+
* says how its text is redacted, and a key an engine sends that the
|
|
106
|
+
* contract does not name is dropped. Every string passes through
|
|
107
|
+
* `redactText`. A secure node keeps no value or selection, and a selection
|
|
108
|
+
* that may show part of a secret is dropped, as a secure field's is. The ref
|
|
109
|
+
* is the engine's own handle, sent back to it to act, never page text.
|
|
116
110
|
*/
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
111
|
+
const NODE_FIELDS = {
|
|
112
|
+
ref: keep,
|
|
113
|
+
role: (role, _node, redaction) => redactText(role, redaction),
|
|
114
|
+
name: (name, _node, redaction) => redactText(name, redaction, OBSERVED_NAME_LIMIT),
|
|
115
|
+
text: (text, _node, redaction) => redactText(text, redaction, OBSERVED_TEXT_LIMIT),
|
|
116
|
+
value: (value, node, redaction) => (node.states?.secure === true ? undefined : redactText(value, redaction, OBSERVED_TEXT_LIMIT)),
|
|
117
|
+
selection: (selection, node, redaction) => node.states?.secure === true || selectionMayHoldSecret(node, selection, redaction)
|
|
118
|
+
? undefined
|
|
119
|
+
: redactText(selection, redaction, OBSERVED_TEXT_LIMIT),
|
|
120
|
+
testId: (testId, _node, redaction) => redactText(testId, redaction),
|
|
121
|
+
inputPurpose: keep,
|
|
122
|
+
states: keep,
|
|
123
|
+
level: keep,
|
|
124
|
+
attributes: (attributes, _node, redaction) => Object.fromEntries(Object.entries(attributes).map(([key, value]) => [redactText(key, redaction), redactText(value, redaction)])),
|
|
125
|
+
rect: keep,
|
|
126
|
+
selector: (selector, _node, redaction) => redactText(selector, redaction),
|
|
127
|
+
framePath: (framePath, _node, redaction) => framePath.map((selector) => redactText(selector, redaction)),
|
|
128
|
+
};
|
|
129
|
+
/**
|
|
130
|
+
* The node and its subtree with every secret taken out of every field
|
|
131
|
+
* (`NODE_FIELDS`), before any consumer reads it: the model's text, the
|
|
132
|
+
* executor's tree, cache descriptors and anchors, failure evidence.
|
|
133
|
+
*/
|
|
134
|
+
export function redactNode(node, redaction) {
|
|
135
|
+
const redacted = {};
|
|
136
|
+
for (const [field, value] of Object.entries(node)) {
|
|
137
|
+
if (field === 'children' || value === undefined || !Object.hasOwn(NODE_FIELDS, field))
|
|
123
138
|
continue;
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
139
|
+
const redactValue = NODE_FIELDS[field];
|
|
140
|
+
const result = redactValue(value, node, redaction);
|
|
141
|
+
if (result !== undefined)
|
|
142
|
+
redacted[field] = result;
|
|
127
143
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
144
|
+
if (node.children !== undefined)
|
|
145
|
+
redacted['children'] = node.children.map((child) => redactNode(child, redaction));
|
|
146
|
+
return redacted;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* An earlier capture's nodes redacted again through the ledger as it is now,
|
|
150
|
+
* each on its own fields, then linked to its redacted-again children, so the
|
|
151
|
+
* tree keeps its shape (which nodes are leaves) without a subtree redacted
|
|
152
|
+
* once per ancestor. A secret a provider resolved after the capture masks
|
|
153
|
+
* text the capture still shows, so the two captures read alike wherever the
|
|
154
|
+
* screen did not change. Redacting redacted text changes nothing else: a
|
|
155
|
+
* marker is never read again.
|
|
156
|
+
*/
|
|
157
|
+
export function redactNodesAgain(nodes, redaction) {
|
|
158
|
+
const again = new Map();
|
|
159
|
+
for (const [id, { children: _children, ...fields }] of nodes)
|
|
160
|
+
again.set(id, redactNode(fields, redaction));
|
|
161
|
+
for (const [id, { children }] of nodes) {
|
|
162
|
+
const node = again.get(id);
|
|
163
|
+
if (node === undefined || children === undefined)
|
|
164
|
+
continue;
|
|
165
|
+
node.children = children.map((child) => again.get(child.ref.id)).filter((child) => child !== undefined);
|
|
166
|
+
}
|
|
167
|
+
return again;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* One observed string with every secret taken out, in the form it is written
|
|
171
|
+
* and in the one-line form a line, a descriptor, or an anchor shows it
|
|
172
|
+
* (`collapseText`). A field exactly as long as `limit`, and so possibly cut
|
|
173
|
+
* there, passes through `redactCut`: a secret the cut stopped partway through
|
|
174
|
+
* leaves a leading part at the end that no whole value matches. A shorter
|
|
175
|
+
* field is whole, and so is a longer one (a native input's value, which no
|
|
176
|
+
* engine cuts); both pass through `redact`. A field whose collapsed form
|
|
177
|
+
* still holds a secret (the whitespace inside a value widened) is kept
|
|
178
|
+
* collapsed and redacted, so no later collapse brings the value back.
|
|
179
|
+
*/
|
|
180
|
+
function redactText(text, redaction, limit) {
|
|
181
|
+
const pass = text.length === limit ? redaction.redactCut : redaction.redact;
|
|
182
|
+
const redacted = pass(text);
|
|
183
|
+
const collapsed = collapseText(redacted);
|
|
184
|
+
if (collapsed === redacted)
|
|
185
|
+
return redacted;
|
|
186
|
+
const read = pass(collapsed);
|
|
187
|
+
return read === collapsed ? redacted : read;
|
|
134
188
|
}
|
|
135
189
|
/**
|
|
136
190
|
* Whether a node's selection may show part of a secret, which no whole-value
|
|
@@ -142,7 +196,7 @@ function protectSecrets(node, redaction) {
|
|
|
142
196
|
function selectionMayHoldSecret(node, selection, redaction) {
|
|
143
197
|
const fields = [node.value, node.text].filter((text) => text !== undefined);
|
|
144
198
|
const cut = (text) => text.length === OBSERVED_TEXT_LIMIT;
|
|
145
|
-
if (fields.some((text) => (
|
|
199
|
+
if (fields.some((text) => redactText(text, redaction, OBSERVED_TEXT_LIMIT) !== text))
|
|
146
200
|
return true;
|
|
147
201
|
return fields.some(cut) && !fields.some((text) => text.includes(selection));
|
|
148
202
|
}
|
|
@@ -234,33 +288,29 @@ export function interactiveNodeCount(observation) {
|
|
|
234
288
|
return count;
|
|
235
289
|
}
|
|
236
290
|
/**
|
|
237
|
-
* Projects
|
|
238
|
-
*
|
|
239
|
-
* selection at all for a secure node, and a link target bounded as a line
|
|
291
|
+
* Projects a redacted tree onto the executor-facing node shape, the fields
|
|
292
|
+
* the text serialization renders, with a link target bounded as a line
|
|
240
293
|
* bounds it: a target on `appOrigin` keeps that origin and loses no more of
|
|
241
294
|
* its path than the line does. Selectors stay behind — they are relocation
|
|
242
295
|
* material for the trace cache, not something a brain reasons about.
|
|
243
296
|
*/
|
|
244
|
-
export function projectTree(node,
|
|
245
|
-
const secure = node.states?.secure === true;
|
|
297
|
+
export function projectTree(node, appOrigin) {
|
|
246
298
|
const attributes = node.attributes === undefined
|
|
247
299
|
? undefined
|
|
248
|
-
: Object.fromEntries(Object.entries(node.attributes).map(([key, value]) => [key, key === 'href' ? treeHref(
|
|
300
|
+
: Object.fromEntries(Object.entries(node.attributes).map(([key, value]) => [key, key === 'href' ? treeHref(value, appOrigin) : value]));
|
|
249
301
|
return {
|
|
250
302
|
id: node.ref.id,
|
|
251
303
|
...(node.role === undefined ? {} : { role: node.role }),
|
|
252
|
-
...(node.name === undefined ? {} : { name:
|
|
253
|
-
...(node.text === undefined ? {} : { text:
|
|
254
|
-
...(node.value === undefined
|
|
255
|
-
...(node.selection === undefined
|
|
304
|
+
...(node.name === undefined ? {} : { name: node.name }),
|
|
305
|
+
...(node.text === undefined ? {} : { text: node.text }),
|
|
306
|
+
...(node.value === undefined ? {} : { value: node.value }),
|
|
307
|
+
...(node.selection === undefined ? {} : { selection: node.selection }),
|
|
256
308
|
...(node.inputPurpose === undefined ? {} : { inputPurpose: node.inputPurpose }),
|
|
257
309
|
...(node.states === undefined ? {} : { states: node.states }),
|
|
258
310
|
...(attributes === undefined ? {} : { attributes }),
|
|
259
311
|
...(node.rect === undefined ? {} : { rect: node.rect }),
|
|
260
312
|
...(node.framePath === undefined ? {} : { framePath: node.framePath }),
|
|
261
|
-
...(node.children === undefined
|
|
262
|
-
? {}
|
|
263
|
-
: { children: node.children.map((child) => projectTree(child, redact, appOrigin)) }),
|
|
313
|
+
...(node.children === undefined ? {} : { children: node.children.map((child) => projectTree(child, appOrigin)) }),
|
|
264
314
|
};
|
|
265
315
|
}
|
|
266
316
|
/** Depth beyond this renders flat; deep chrome must not buy tokens with spaces. */
|
|
@@ -273,37 +323,37 @@ const MAX_HREF_LENGTH = 256;
|
|
|
273
323
|
* lines, and the model needs their text, not a filler word. A link target on
|
|
274
324
|
* `appOrigin`, the app base URL's, renders as its path.
|
|
275
325
|
*/
|
|
276
|
-
export function formatNode(node, depth,
|
|
326
|
+
export function formatNode(node, depth, appOrigin) {
|
|
277
327
|
const parts = [`#${node.ref.id}`];
|
|
278
328
|
if (node.role !== undefined && node.role !== '')
|
|
279
329
|
parts.push(node.role);
|
|
280
330
|
if (node.name !== undefined && node.name !== '')
|
|
281
|
-
parts.push(JSON.stringify(
|
|
331
|
+
parts.push(JSON.stringify(node.name));
|
|
282
332
|
const text = node.text === undefined ? '' : collapseText(node.text);
|
|
283
333
|
if (text !== '' && text !== node.name)
|
|
284
|
-
parts.push(`text=${JSON.stringify(
|
|
334
|
+
parts.push(`text=${JSON.stringify(text)}`);
|
|
285
335
|
// Disambiguators the model needs when role and name repeat. The engine has
|
|
286
336
|
// already reduced href to origin and path.
|
|
287
337
|
if (node.testId !== undefined && node.testId !== '')
|
|
288
338
|
parts.push(`testid=${JSON.stringify(node.testId)}`);
|
|
289
339
|
const href = node.attributes?.['href'];
|
|
290
340
|
if (href !== undefined && href !== '')
|
|
291
|
-
parts.push(`href=${JSON.stringify(renderHref(
|
|
341
|
+
parts.push(`href=${JSON.stringify(renderHref(href, appOrigin))}`);
|
|
292
342
|
const placeholder = node.attributes?.['placeholder'];
|
|
293
343
|
if (placeholder !== undefined && placeholder !== '' && (node.name ?? '') === '') {
|
|
294
|
-
parts.push(`placeholder=${JSON.stringify(
|
|
344
|
+
parts.push(`placeholder=${JSON.stringify(placeholder)}`);
|
|
295
345
|
}
|
|
296
346
|
if (node.states?.secure === true) {
|
|
297
347
|
parts.push('value=<secure>');
|
|
298
348
|
}
|
|
299
349
|
else if (node.value !== undefined && node.value !== '') {
|
|
300
|
-
parts.push(`value=${JSON.stringify(
|
|
350
|
+
parts.push(`value=${JSON.stringify(node.value)}`);
|
|
301
351
|
}
|
|
302
352
|
// What a Shift+Arrow press selected: without it the model extends a
|
|
303
353
|
// selection blind and cannot tell one word from its neighbour. Rendered
|
|
304
354
|
// verbatim (escaped), since a selected space or newline is a selection too.
|
|
305
|
-
if (node.selection !== undefined && node.selection !== ''
|
|
306
|
-
parts.push(`selection=${JSON.stringify(
|
|
355
|
+
if (node.selection !== undefined && node.selection !== '') {
|
|
356
|
+
parts.push(`selection=${JSON.stringify(node.selection)}`);
|
|
307
357
|
}
|
|
308
358
|
if (node.inputPurpose !== undefined && node.inputPurpose !== 'none') {
|
|
309
359
|
parts.push(`purpose=${node.inputPurpose}`);
|
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
* screenshot are scaled into the observation's CSS pixels first.
|
|
8
8
|
*/
|
|
9
9
|
import type { PointActionName } from '../cache/trace.ts';
|
|
10
|
-
import type {
|
|
10
|
+
import type { ViewportPoint, ViewportSize } from '../engine/surface.ts';
|
|
11
11
|
import type { ExecutorPixels } from './executor.ts';
|
|
12
|
-
import { type AgentObservation } from './observation.ts';
|
|
12
|
+
import { type AgentObservation, type RedactedNode } from './observation.ts';
|
|
13
13
|
/**
|
|
14
14
|
* Scales a point read off a screenshot into the observation's CSS pixels and
|
|
15
15
|
* clamps it to the viewport. `pixels.scale` is image pixels per CSS pixel
|
|
@@ -23,9 +23,9 @@ export declare function imagePointToViewport(point: {
|
|
|
23
23
|
}, pixels: Pick<ExecutorPixels, 'width' | 'height' | 'scale'>, viewport: ViewportSize): ViewportPoint;
|
|
24
24
|
export interface HitTest {
|
|
25
25
|
/** The innermost enabled control whose box contains the point, if any. */
|
|
26
|
-
readonly control:
|
|
26
|
+
readonly control: RedactedNode | undefined;
|
|
27
27
|
/** The innermost listed node of any role whose box contains the point, if any. */
|
|
28
|
-
readonly under:
|
|
28
|
+
readonly under: RedactedNode | undefined;
|
|
29
29
|
}
|
|
30
30
|
/**
|
|
31
31
|
* Finds what the newest observation lists at a viewport point. Only an
|
|
@@ -54,12 +54,12 @@ export declare const POINT_VERBS: Readonly<Record<PointActionName, {
|
|
|
54
54
|
export declare function describePointAction(input: PointProse & {
|
|
55
55
|
readonly verb: PointActionName;
|
|
56
56
|
readonly point: ViewportPoint;
|
|
57
|
-
readonly control:
|
|
58
|
-
readonly under:
|
|
57
|
+
readonly control: RedactedNode | undefined;
|
|
58
|
+
readonly under: RedactedNode | undefined;
|
|
59
59
|
}): string;
|
|
60
60
|
/** What one hit test found, for the model that named the point. */
|
|
61
61
|
export declare function describePointHit(input: PointProse & {
|
|
62
62
|
readonly point: ViewportPoint;
|
|
63
|
-
readonly control:
|
|
64
|
-
readonly under:
|
|
63
|
+
readonly control: RedactedNode | undefined;
|
|
64
|
+
readonly under: RedactedNode | undefined;
|
|
65
65
|
}): string;
|
package/dist/agent/point-tap.js
CHANGED
|
@@ -41,7 +41,7 @@ export function hitTest(observation, point) {
|
|
|
41
41
|
* The innermost listed node whose box contains the point, among those
|
|
42
42
|
* `accept` admits: deepest in the tree, then the smallest box. Hidden nodes
|
|
43
43
|
* and nodes without a box are skipped. Every box is in the top-level
|
|
44
|
-
* viewport's CSS pixels, nested documents included (`
|
|
44
|
+
* viewport's CSS pixels, nested documents included (`RedactedNode.rect`), so
|
|
45
45
|
* a node inside an iframe is found like any other.
|
|
46
46
|
*/
|
|
47
47
|
function innermostAt(observation, point, accept = () => true) {
|
package/dist/agent/replay.d.ts
CHANGED
|
@@ -11,15 +11,20 @@
|
|
|
11
11
|
* timeout, cancellation) rethrow untouched; they are the step's truth, not a
|
|
12
12
|
* divergence.
|
|
13
13
|
*/
|
|
14
|
-
import { type ActionTrace, type DerivedReason, type
|
|
15
|
-
import type {
|
|
14
|
+
import { type ActionTrace, type DerivedReason, type TraceViewport } from '../cache/trace.ts';
|
|
15
|
+
import type { RedactedNode } from './observation.ts';
|
|
16
16
|
import { type ExecutorActions, type ReplayHandOffReason } from './executor.ts';
|
|
17
17
|
import { type SettleMode } from './settle-policy.ts';
|
|
18
18
|
/** The nodes of one observation, keyed by their per-observation ids. */
|
|
19
|
-
export type ObservedNodes = ReadonlyMap<string,
|
|
19
|
+
export type ObservedNodes = ReadonlyMap<string, RedactedNode>;
|
|
20
20
|
/** One capture's location and viewport, with nodes only when semantic evidence is available. */
|
|
21
21
|
export type ObservedScreen = {
|
|
22
22
|
readonly viewport: TraceViewport;
|
|
23
|
+
/**
|
|
24
|
+
* Where the capture was, as the cache compares it (`appLocation`): on the
|
|
25
|
+
* app's own origin its path, query, and fragment; elsewhere the whole
|
|
26
|
+
* location; a device's screen title as it is.
|
|
27
|
+
*/
|
|
23
28
|
readonly path?: string;
|
|
24
29
|
} & ({
|
|
25
30
|
readonly kind: 'semantic';
|
|
@@ -45,7 +50,6 @@ export interface ReplayHost {
|
|
|
45
50
|
readonly actions: ExecutorActions;
|
|
46
51
|
readonly signal: AbortSignal;
|
|
47
52
|
remainingMs(): number;
|
|
48
|
-
readonly redact: (text: string) => string;
|
|
49
53
|
}
|
|
50
54
|
export interface ReplayOutcome {
|
|
51
55
|
/** True when every recorded action executed; the step self-finalizes. */
|
|
@@ -67,20 +71,27 @@ export interface ReplayOptions {
|
|
|
67
71
|
* instead of capturing again: nothing has happened since it was taken.
|
|
68
72
|
*/
|
|
69
73
|
readonly initial?: SemanticScreen;
|
|
74
|
+
/**
|
|
75
|
+
* Asked before a free action (a navigate, typed text, a key) that follows
|
|
76
|
+
* another action; true takes the look a targeted action would have taken.
|
|
77
|
+
* A free action reads no screen, so without it the screen the previous
|
|
78
|
+
* action left goes unseen.
|
|
79
|
+
*/
|
|
80
|
+
readonly looksBeforeFree?: () => boolean;
|
|
70
81
|
}
|
|
71
82
|
/** Replays one trace until it completes or diverges. */
|
|
72
83
|
export declare function replayTrace(host: ReplayHost, trace: ActionTrace, options?: ReplayOptions): Promise<ReplayOutcome>;
|
|
73
84
|
/**
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
85
|
+
* Waits for a trace's recorded end state on the live screen: `holds` (the
|
|
86
|
+
* recorded delta, `cache/anchors.ts`) must be true of one look, or the
|
|
87
|
+
* replay must not pass on its own. Waits on the same settling backoff
|
|
77
88
|
* relocation uses, because the recording run's final look came seconds of
|
|
78
89
|
* model latency after its last action and a replay's comes right away: a
|
|
79
90
|
* save still in flight is a wait, not a divergence. A surface that cannot be
|
|
80
91
|
* observed at all is a mismatch too — the executor gets the step and judges
|
|
81
92
|
* the live state; only runtime hard stops propagate.
|
|
82
93
|
*/
|
|
83
|
-
export declare function
|
|
94
|
+
export declare function verifyEndState(host: ReplayHost, holds: (screen: SemanticScreen) => boolean, options?: {
|
|
84
95
|
readonly waitMs?: number;
|
|
85
96
|
readonly initial?: SemanticScreen;
|
|
86
97
|
}): Promise<boolean>;
|
package/dist/agent/replay.js
CHANGED
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
* timeout, cancellation) rethrow untouched; they are the step's truth, not a
|
|
12
12
|
* divergence.
|
|
13
13
|
*/
|
|
14
|
-
import { anchorsPresent } from '../cache/anchors.js';
|
|
15
14
|
import { isRelocatableDescriptor, MAIN_LIST_SHARE, relocateDescriptor } from '../cache/relocate.js';
|
|
16
15
|
import { isNodeAction } from '../cache/trace.js';
|
|
17
16
|
import { hasCause } from '../internal/errors.js';
|
|
@@ -146,6 +145,8 @@ export async function replayTrace(host, trace, options = {}) {
|
|
|
146
145
|
break;
|
|
147
146
|
}
|
|
148
147
|
case 'free':
|
|
148
|
+
if (previous !== undefined && options.looksBeforeFree?.() === true)
|
|
149
|
+
await firstLook(host, look);
|
|
149
150
|
await planned.invoke();
|
|
150
151
|
break;
|
|
151
152
|
case 'scroll': {
|
|
@@ -228,21 +229,19 @@ export async function replayTrace(host, trace, options = {}) {
|
|
|
228
229
|
return { completed: true, executed: summaries.length, total, summaries };
|
|
229
230
|
}
|
|
230
231
|
/**
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
*
|
|
232
|
+
* Waits for a trace's recorded end state on the live screen: `holds` (the
|
|
233
|
+
* recorded delta, `cache/anchors.ts`) must be true of one look, or the
|
|
234
|
+
* replay must not pass on its own. Waits on the same settling backoff
|
|
234
235
|
* relocation uses, because the recording run's final look came seconds of
|
|
235
236
|
* model latency after its last action and a replay's comes right away: a
|
|
236
237
|
* save still in flight is a wait, not a divergence. A surface that cannot be
|
|
237
238
|
* observed at all is a mismatch too — the executor gets the step and judges
|
|
238
239
|
* the live state; only runtime hard stops propagate.
|
|
239
240
|
*/
|
|
240
|
-
export async function
|
|
241
|
-
if (anchors.length === 0)
|
|
242
|
-
return true;
|
|
241
|
+
export async function verifyEndState(host, holds, options = {}) {
|
|
243
242
|
const startedMs = Date.now();
|
|
244
243
|
try {
|
|
245
|
-
const present = await pollSettled(host, (
|
|
244
|
+
const present = await pollSettled(host, (screen) => holds(screen) ? true : undefined, options.initial === undefined ? HELD_STILL : { kind: 'in-hand', screen: options.initial });
|
|
246
245
|
if (present === true)
|
|
247
246
|
return true;
|
|
248
247
|
// The settling backoff covers a slow re-render; the recorded run may have
|
|
@@ -255,7 +254,7 @@ export async function verifyAnchors(host, anchors, options = {}) {
|
|
|
255
254
|
const screen = await host.observe('raw');
|
|
256
255
|
if (screen.kind === 'pixels' || !host.traceEligible)
|
|
257
256
|
return false;
|
|
258
|
-
if (
|
|
257
|
+
if (holds(screen))
|
|
259
258
|
return true;
|
|
260
259
|
}
|
|
261
260
|
return false;
|
|
@@ -272,10 +271,9 @@ const END_WAIT_POLL_MS = 1_000;
|
|
|
272
271
|
const END_WAIT_RESERVE_MS = 20_000;
|
|
273
272
|
/** Relocates one descriptor against the settling screen, looking again while `retryable` says the failure may pass. */
|
|
274
273
|
async function relocate(host, descriptor, look) {
|
|
275
|
-
const options = { redact: host.redact };
|
|
276
274
|
let last = { kind: 'failed', failure: 'target-not-found' };
|
|
277
275
|
const settled = await pollSettled(host, (screen) => {
|
|
278
|
-
const result = relocateDescriptor(descriptor, screen.nodes
|
|
276
|
+
const result = relocateDescriptor(descriptor, screen.nodes);
|
|
279
277
|
if (result.kind === 'failed') {
|
|
280
278
|
last = result.failure === 'target-not-found' ? result : { ...result, screen };
|
|
281
279
|
return retryable(result.failure, descriptor) ? undefined : last;
|
|
@@ -294,17 +292,16 @@ async function relocate(host, descriptor, look) {
|
|
|
294
292
|
* unpositioned ambiguity is not.
|
|
295
293
|
*/
|
|
296
294
|
async function relocatePair(host, source, destination, look) {
|
|
297
|
-
const options = { redact: host.redact };
|
|
298
295
|
let last = 'target-not-found';
|
|
299
296
|
const failed = (descriptor, failure) => {
|
|
300
297
|
last = failure;
|
|
301
298
|
return retryable(failure, descriptor) ? undefined : { kind: 'failed', failure };
|
|
302
299
|
};
|
|
303
300
|
const settled = await pollSettled(host, (screen) => {
|
|
304
|
-
const from = relocateDescriptor(source, screen.nodes
|
|
301
|
+
const from = relocateDescriptor(source, screen.nodes);
|
|
305
302
|
if (from.kind === 'failed')
|
|
306
303
|
return failed(source, from.failure);
|
|
307
|
-
const to = relocateDescriptor(destination, screen.nodes
|
|
304
|
+
const to = relocateDescriptor(destination, screen.nodes);
|
|
308
305
|
if (to.kind === 'failed')
|
|
309
306
|
return failed(destination, to.failure);
|
|
310
307
|
return { kind: 'found', source: from.id, destination: to.id };
|
|
@@ -10,11 +10,10 @@
|
|
|
10
10
|
* all scrolls as the viewport does, which is what scrolling the main list
|
|
11
11
|
* does; a smaller region that vanished is gone, as for any other action.
|
|
12
12
|
*/
|
|
13
|
-
import type { SemanticNode } from '../engine/surface.ts';
|
|
14
13
|
import { type Box } from '../internal/geometry.ts';
|
|
15
14
|
import type { ExecutorTarget } from './executor.ts';
|
|
16
15
|
import { type ObservationFeed, type Resolved } from './observation-feed.ts';
|
|
17
|
-
import type { SemanticAgentObservation } from './observation.ts';
|
|
16
|
+
import type { RedactedNode, SemanticAgentObservation } from './observation.ts';
|
|
18
17
|
/**
|
|
19
18
|
* The node a scroll target names now: the node itself, its descriptor's
|
|
20
19
|
* match, or the same-role node in its old place. Undefined for a lost list
|
|
@@ -22,4 +21,4 @@ import type { SemanticAgentObservation } from './observation.ts';
|
|
|
22
21
|
*/
|
|
23
22
|
export declare function resolveScrollTarget(feed: ObservationFeed, target: ExecutorTarget): Resolved | undefined;
|
|
24
23
|
/** The listed node of the role whose box overlaps `box` most, when it overlaps enough. */
|
|
25
|
-
export declare function samePlace(observation: SemanticAgentObservation, role: string | undefined, box: Box):
|
|
24
|
+
export declare function samePlace(observation: SemanticAgentObservation, role: string | undefined, box: Box): RedactedNode | undefined;
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* when replay performed no action.
|
|
13
13
|
*/
|
|
14
14
|
import type { AgentCacheContext } from '../cache/context.ts';
|
|
15
|
+
import type { CacheAgentIdentity } from '../cache/identity.ts';
|
|
15
16
|
import { type ParamTemplate } from '../cache/template.ts';
|
|
16
17
|
import { type DerivedReason } from '../cache/trace.ts';
|
|
17
18
|
import type { StepCacheInfo } from '../run/steps.ts';
|
|
@@ -42,7 +43,11 @@ export interface StepCacheOptions {
|
|
|
42
43
|
readonly name: string;
|
|
43
44
|
readonly version?: string;
|
|
44
45
|
};
|
|
46
|
+
/** The agent the step runs with, part of the key (`cache/identity.ts`). */
|
|
47
|
+
readonly agent: CacheAgentIdentity;
|
|
45
48
|
readonly redact: (text: string) => string;
|
|
49
|
+
/** `redact` for a field cut at its observed limit (`SecretLedger.redactCut`). */
|
|
50
|
+
readonly redactCut: (text: string) => string;
|
|
46
51
|
readonly maxActions: number;
|
|
47
52
|
/** Timeline index of the step being dispatched. */
|
|
48
53
|
readonly stepIndex: number;
|
|
@@ -60,8 +65,11 @@ export declare class StepTraceSession {
|
|
|
60
65
|
private readonly host;
|
|
61
66
|
private readonly cache;
|
|
62
67
|
private readonly keyHash;
|
|
68
|
+
/** The claimed key and the step it names, which an entry records as its provenance. */
|
|
69
|
+
private readonly claim;
|
|
63
70
|
private readonly recorder;
|
|
64
71
|
private readonly options;
|
|
72
|
+
private readonly redaction;
|
|
65
73
|
private info;
|
|
66
74
|
private prefix;
|
|
67
75
|
private startPath;
|
|
@@ -85,6 +93,8 @@ export declare class StepTraceSession {
|
|
|
85
93
|
private replayedWhole;
|
|
86
94
|
/** True once a cached entry's actions were run this step, fully or partly. */
|
|
87
95
|
private consumedReplay;
|
|
96
|
+
/** True once the store returned an entry for this step, whether or not it replayed. */
|
|
97
|
+
private readEntryHit;
|
|
88
98
|
/** True once `cache.strict` failed the step on its recording, which is then kept for review rather than evicted. */
|
|
89
99
|
private failedStale;
|
|
90
100
|
/** Grammar actions recorded so far when an end-mismatch hand-off happened. */
|
|
@@ -117,6 +127,14 @@ export declare class StepTraceSession {
|
|
|
117
127
|
* model call to absorb on every run. The report keeps the cache detail.
|
|
118
128
|
*/
|
|
119
129
|
private failIfStale;
|
|
130
|
+
/**
|
|
131
|
+
* Under `cache.strict`, ends a step whose key found no entry while the
|
|
132
|
+
* store holds a recording made for the same step under another key: the
|
|
133
|
+
* runner, the engine, the app, or the agent's context changed since, and
|
|
134
|
+
* the recording no longer replays as surely as one that diverged. A step
|
|
135
|
+
* whose instruction or params changed is a new step and still runs live.
|
|
136
|
+
*/
|
|
137
|
+
private failIfRekeyed;
|
|
120
138
|
/**
|
|
121
139
|
* Closes the session once the step has settled. The whole write-side
|
|
122
140
|
* decision lives here, in read-write mode only:
|
|
@@ -136,6 +154,10 @@ export declare class StepTraceSession {
|
|
|
136
154
|
* anchor, or end wait, so drift is repaired only once a relocation fails
|
|
137
155
|
* and the hand-off that follows re-records.
|
|
138
156
|
* - passed otherwise: stage the recorded trace for attempt-end settlement.
|
|
157
|
+
* When this pass leaves nothing to stage (it changed nothing a replay
|
|
158
|
+
* could check) and an entry was read for the step, evict that entry: it
|
|
159
|
+
* did not serve this pass, and nothing recorded will replace it, so
|
|
160
|
+
* keeping it would hand every later run off the same way.
|
|
139
161
|
* - failed after consuming a replay: evict. Without this, a diverged replay
|
|
140
162
|
* whose step then fails stages nothing — and the poisoned entry would
|
|
141
163
|
* replay its bad prefix on every future first attempt.
|
|
@@ -172,18 +194,22 @@ export declare class StepTraceSession {
|
|
|
172
194
|
private replayEntry;
|
|
173
195
|
/**
|
|
174
196
|
* The trace's postcondition against the live screen: the recorded end
|
|
175
|
-
* route
|
|
176
|
-
*
|
|
177
|
-
*
|
|
197
|
+
* route when the live location is known, the recorded delta on the
|
|
198
|
+
* screen (`deltaHolds`), and evidence that the replay produced it rather
|
|
199
|
+
* than finding it there (`deltaEvidenced`). The evidence is measured from
|
|
200
|
+
* the first screen the replay saw on the route it ended on (`screens`, in
|
|
201
|
+
* order): the start for a step that stays on one screen, the page a
|
|
202
|
+
* recorded navigate opened before the switch on it was tapped. An outcome
|
|
203
|
+
* already showing there proves nothing, and neither does a recording with
|
|
204
|
+
* no delta at all.
|
|
178
205
|
*/
|
|
179
206
|
private endStateMatches;
|
|
180
207
|
/**
|
|
181
208
|
* Captures the semantic end state once its route matches the recording,
|
|
182
|
-
* polling while a navigation the last action started commits.
|
|
183
|
-
*
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
* the recording, is the one that decides it, as for a route that matched.
|
|
209
|
+
* polling while a navigation the last action started commits. A route
|
|
210
|
+
* that never matches is another screen, whatever it shows: a link that
|
|
211
|
+
* now lands on a page sharing the recorded one's layout is exactly the
|
|
212
|
+
* flow that went wrong.
|
|
187
213
|
*/
|
|
188
214
|
private endScreen;
|
|
189
215
|
private selfFinalize;
|
|
@@ -199,12 +225,13 @@ export declare class StepTraceSession {
|
|
|
199
225
|
*
|
|
200
226
|
* The end path and a fresh settled observation are the trace's
|
|
201
227
|
* postcondition — the state the step passed in. The delta between the
|
|
202
|
-
* baseline and the passing observation becomes the
|
|
203
|
-
* step moved to another pathname the whole new
|
|
204
|
-
* first stable anchors back the path check,
|
|
205
|
-
* record's page up to the id the app minted for
|
|
206
|
-
* cannot be captured stages nothing
|
|
207
|
-
* replay
|
|
228
|
+
* baseline and the passing observation becomes the anchors: what appeared
|
|
229
|
+
* and what vanished. When the step moved to another pathname the whole new
|
|
230
|
+
* screen is the delta, and its first stable anchors back the path check,
|
|
231
|
+
* which matches a created record's page up to the id the app minted for
|
|
232
|
+
* it. A postcondition that cannot be captured stages nothing, and neither
|
|
233
|
+
* does a step that changed nothing a replay could check, no node and no
|
|
234
|
+
* route: a trace without its check would replay on mechanics alone.
|
|
208
235
|
*/
|
|
209
236
|
private stage;
|
|
210
237
|
/**
|