e2e 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build.json +1 -1
- package/dist/agent/act.js +7 -4
- package/dist/agent/action-dispatcher.js +4 -5
- package/dist/agent/actions.d.ts +32 -23
- package/dist/agent/actions.js +38 -24
- package/dist/agent/model/sdk.js +2 -0
- package/dist/agent/observation-feed.d.ts +5 -6
- package/dist/agent/observation-feed.js +5 -7
- package/dist/agent/observation.d.ts +40 -11
- package/dist/agent/observation.js +109 -59
- package/dist/agent/point-tap.d.ts +8 -8
- package/dist/agent/point-tap.js +1 -1
- package/dist/agent/replay.d.ts +19 -8
- package/dist/agent/replay.js +11 -14
- package/dist/agent/scroll-target.d.ts +2 -3
- package/dist/agent/step-cache.d.ts +41 -14
- package/dist/agent/step-cache.js +147 -62
- package/dist/agent/tool-loop.js +2 -0
- package/dist/cache/anchors.d.ts +72 -23
- package/dist/cache/anchors.js +233 -60
- package/dist/cache/context.d.ts +25 -14
- package/dist/cache/context.js +50 -8
- package/dist/cache/decide.d.ts +2 -2
- package/dist/cache/decide.js +4 -6
- package/dist/cache/identity.d.ts +18 -4
- package/dist/cache/identity.js +9 -4
- package/dist/cache/recorder.d.ts +15 -1
- package/dist/cache/recorder.js +36 -23
- package/dist/cache/rekeyed.d.ts +46 -0
- package/dist/cache/rekeyed.js +89 -0
- package/dist/cache/relocate.d.ts +14 -14
- package/dist/cache/relocate.js +33 -25
- package/dist/cache/route.d.ts +32 -33
- package/dist/cache/route.js +94 -50
- package/dist/cache/store.d.ts +6 -0
- package/dist/cache/store.js +21 -1
- package/dist/cache/trace.d.ts +53 -14
- package/dist/cache/trace.js +70 -28
- package/dist/cli/index.js +8 -8
- package/dist/cli/init/engines.d.ts +1 -3
- package/dist/cli/init/engines.js +2 -2
- package/dist/cli/init/gateways.d.ts +1 -1
- package/dist/cli/init/gateways.js +11 -1
- package/dist/cli/init/sibling-versions.json +1 -1
- package/dist/cli/init/versions.d.ts +0 -8
- package/dist/cli/init/versions.js +1 -10
- package/dist/config/resolve.d.ts +1 -0
- package/dist/config/resolve.js +33 -1
- package/dist/engine/index.d.ts +2 -2
- package/dist/expect/async.js +2 -1
- package/dist/expect/poll-scope.d.ts +52 -0
- package/dist/expect/poll-scope.js +113 -0
- package/dist/expect/poll.js +31 -4
- package/dist/expect/values.js +11 -4
- package/dist/internal/abandoned.d.ts +11 -0
- package/dist/internal/abandoned.js +23 -0
- package/dist/internal/brands.d.ts +2 -0
- package/dist/internal/brands.js +2 -0
- package/dist/internal/client-identity.d.ts +9 -0
- package/dist/internal/client-identity.js +14 -0
- package/dist/internal/redact.d.ts +7 -4
- package/dist/internal/redact.js +116 -23
- package/dist/internal/time.d.ts +6 -0
- package/dist/internal/time.js +25 -13
- package/dist/internal/urls.d.ts +9 -5
- package/dist/internal/urls.js +19 -10
- package/dist/locator/engine.d.ts +12 -2
- package/dist/locator/engine.js +8 -3
- package/dist/locator/screen.js +28 -4
- package/dist/mcp/catalog.d.ts +3 -2
- package/dist/mcp/catalog.js +6 -6
- package/dist/mcp/session.js +1 -1
- package/dist/oauth/chatgpt.js +3 -23
- package/dist/oauth/cli.js +1 -0
- package/dist/oauth/copilot.d.ts +5 -4
- package/dist/oauth/copilot.js +80 -7
- package/dist/oauth/grok.js +2 -1
- package/dist/oauth/models.js +2 -1
- package/dist/oauth/opencode-console.d.ts +10 -0
- package/dist/oauth/opencode-console.js +133 -0
- package/dist/oauth/providers/github-copilot.d.ts +12 -0
- package/dist/oauth/providers/github-copilot.js +77 -23
- package/dist/oauth/providers/opencode-console.d.ts +31 -0
- package/dist/oauth/providers/opencode-console.js +148 -0
- package/dist/oauth/providers.d.ts +2 -6
- package/dist/oauth/providers.js +3 -7
- package/dist/oauth/responses.d.ts +9 -0
- package/dist/oauth/responses.js +23 -0
- package/dist/report/build.d.ts +26 -0
- package/dist/report/build.js +16 -8
- package/dist/report/failure-text.d.ts +6 -1
- package/dist/report/failure-text.js +14 -4
- package/dist/report/format.d.ts +10 -5
- package/dist/report/format.js +12 -5
- package/dist/report/junit.js +11 -12
- package/dist/report/list.d.ts +1 -0
- package/dist/report/list.js +38 -18
- package/dist/report/markdown.d.ts +1 -1
- package/dist/report/markdown.js +37 -14
- package/dist/report/repeats.d.ts +6 -1
- package/dist/report/repeats.js +12 -3
- package/dist/run/artifacts.d.ts +18 -0
- package/dist/run/artifacts.js +68 -1
- package/dist/run/execute.d.ts +9 -0
- package/dist/run/execute.js +49 -24
- package/dist/run/extended-fixtures.js +6 -2
- package/dist/run/failure-evidence.js +4 -2
- package/dist/run/fixtures.js +6 -8
- package/dist/run/in-process.d.ts +2 -0
- package/dist/run/in-process.js +1 -0
- package/dist/run/last-run.d.ts +42 -8
- package/dist/run/last-run.js +121 -14
- package/dist/run/realm.d.ts +4 -1
- package/dist/run/realm.js +20 -6
- package/dist/run/records.d.ts +16 -0
- package/dist/run/records.js +26 -0
- package/dist/run/retry.d.ts +3 -1
- package/dist/run/retry.js +4 -2
- package/dist/run/runner.d.ts +1 -1
- package/dist/run/runner.js +69 -30
- package/dist/run/serial.d.ts +2 -0
- package/dist/run/serial.js +8 -5
- package/dist/run/steps.d.ts +0 -6
- package/dist/run/steps.js +3 -20
- package/dist/run/worker/entry.js +5 -4
- package/dist/run/worker/protocol.d.ts +2 -0
- package/dist/run/worker/session.d.ts +2 -0
- package/dist/run/worker/session.js +1 -0
- package/dist/telemetry/events.js +1 -0
- package/dist/types.d.ts +9 -3
- package/docs/agents.mdx +6 -3
- package/docs/assertions.mdx +94 -0
- package/docs/authentication.mdx +3 -1
- package/docs/browser.mdx +42 -6
- package/docs/cache.mdx +64 -18
- package/docs/ci.mdx +11 -7
- package/docs/debugging.mdx +6 -3
- package/docs/executors.mdx +4 -2
- package/docs/github.mdx +12 -4
- package/docs/{writing-tests.mdx → goals.mdx} +21 -101
- package/docs/index.mdx +4 -4
- package/docs/integrations/kernel.mdx +1 -1
- package/docs/locators.mdx +64 -0
- package/docs/migrate/cypress.mdx +2 -2
- package/docs/migrate/maestro.mdx +2 -2
- package/docs/migrate/playwright.mdx +19 -16
- package/docs/migrate/selenium.mdx +2 -2
- package/docs/mobile.mdx +21 -4
- package/docs/models.mdx +53 -7
- package/docs/physical-devices.mdx +76 -0
- package/docs/quickstart.mdx +4 -3
- package/docs/reference/app.mdx +1 -1
- package/docs/reference/cli.mdx +30 -13
- package/docs/reference/config.mdx +25 -7
- package/docs/reference/engine.mdx +3 -3
- package/docs/reference/environment.mdx +8 -1
- package/docs/reference/errors.mdx +5 -5
- package/docs/reference/expect.mdx +34 -12
- package/docs/reference/mobile.mdx +23 -13
- package/docs/reference/reporters.mdx +8 -2
- package/docs/reference/screen.mdx +30 -7
- package/docs/reference/test.mdx +11 -4
- package/docs/reference/web.mdx +125 -21
- package/docs/security.mdx +26 -16
- package/docs/subscriptions.mdx +91 -12
- package/docs/telemetry.mdx +1 -1
- package/package.json +16 -3
- package/schema/report-v1.schema.json +14 -1
- package/skills/e2e/SKILL.md +2 -2
- package/skills/e2e/references/agent.md +22 -11
- package/skills/e2e/references/debugging.md +4 -4
- package/skills/e2e/references/running.md +13 -8
- package/skills/e2e/references/setup.md +36 -15
- package/skills/e2e/references/writing-tests.md +34 -17
package/dist/.build.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":"0.
|
|
1
|
+
{"version":"0.17.0"}
|
package/dist/agent/act.js
CHANGED
|
@@ -29,6 +29,7 @@ import { projectPriorSteps, serializeLedger } from './ledger.js';
|
|
|
29
29
|
import { instantiateLanguageModel } from './model/sdk.js';
|
|
30
30
|
import { ObservationFeed } from './observation-feed.js';
|
|
31
31
|
import { recordPolicyEvent } from './phases.js';
|
|
32
|
+
import { appLocation } from '../cache/route.js';
|
|
32
33
|
import { OperationQueue } from './operation-queue.js';
|
|
33
34
|
import { StepAccounting } from './step-accounting.js';
|
|
34
35
|
import { failedStepOutcome, StepTraceSession } from './step-cache.js';
|
|
@@ -199,7 +200,10 @@ class ActDispatch {
|
|
|
199
200
|
name: agent.executor.name,
|
|
200
201
|
...(agent.executor.version === undefined ? {} : { version: agent.executor.version }),
|
|
201
202
|
},
|
|
203
|
+
// Redacted already (`dispatchAgentStep`): the key digests the context the executor reads.
|
|
204
|
+
agent: { name: agent.name, context: agent.agentContext },
|
|
202
205
|
redact: runtime.redact,
|
|
206
|
+
redactCut: runtime.redactCut,
|
|
203
207
|
maxActions: this.accounting.maxActions,
|
|
204
208
|
stepIndex,
|
|
205
209
|
});
|
|
@@ -421,11 +425,10 @@ class ActDispatch {
|
|
|
421
425
|
const feed = this.feed;
|
|
422
426
|
return {
|
|
423
427
|
get traceEligible() { return feed.traceEligible; },
|
|
424
|
-
observe: async (mode) => screenOf(await this.feed.probe(mode)),
|
|
428
|
+
observe: async (mode) => screenOf(await this.feed.probe(mode), this.runtime.app.base?.origin),
|
|
425
429
|
actions: this.dispatcher.actions,
|
|
426
430
|
signal: this.accounting.signal,
|
|
427
431
|
remainingMs: () => this.accounting.remainingMs(),
|
|
428
|
-
redact: this.runtime.redact,
|
|
429
432
|
replaying: (active) => {
|
|
430
433
|
this.runtime.steps.replaying(active);
|
|
431
434
|
this.accounting.replaying(active);
|
|
@@ -474,10 +477,10 @@ class ActDispatch {
|
|
|
474
477
|
}
|
|
475
478
|
}
|
|
476
479
|
/** The replay engine's view of a capture: the nodes, and the viewport a recorded point is checked against. */
|
|
477
|
-
function screenOf(observation) {
|
|
480
|
+
function screenOf(observation, appOrigin) {
|
|
478
481
|
const metadata = {
|
|
479
482
|
viewport: { width: observation.viewport.width, height: observation.viewport.height },
|
|
480
|
-
...(observation.
|
|
483
|
+
...(observation.location === undefined ? {} : { path: appLocation(observation.location, appOrigin) }),
|
|
481
484
|
};
|
|
482
485
|
return observation.kind === 'semantic'
|
|
483
486
|
? { ...metadata, kind: 'semantic', nodes: observation.nodes }
|
|
@@ -487,10 +487,10 @@ export class ActionDispatcher {
|
|
|
487
487
|
}
|
|
488
488
|
async runActionNow(name, body) {
|
|
489
489
|
this.accounting.reserveAction();
|
|
490
|
-
const
|
|
490
|
+
const redaction = { redact: this.runtime.redact, redactCut: this.runtime.redactCut };
|
|
491
491
|
let action;
|
|
492
492
|
try {
|
|
493
|
-
action = await instrumentPhase(this.runtime, { api: this.accounting.api, kind: 'engine', phase: 'agent.action', name }, body, (committed) => ({ detail: describeAction(committed,
|
|
493
|
+
action = await instrumentPhase(this.runtime, { api: this.accounting.api, kind: 'engine', phase: 'agent.action', name }, body, (committed) => ({ detail: describeAction(committed, redaction).summary }));
|
|
494
494
|
}
|
|
495
495
|
catch (cause) {
|
|
496
496
|
this.accounting.checkpoint(cause);
|
|
@@ -579,9 +579,8 @@ export class ActionDispatcher {
|
|
|
579
579
|
* number picks the same one.
|
|
580
580
|
*/
|
|
581
581
|
placementOf({ node, observation }) {
|
|
582
|
-
const
|
|
583
|
-
const
|
|
584
|
-
const position = describePosition(node, within, observation.nodes, { redact });
|
|
582
|
+
const within = containerKey(node.ref.id, observation.nodes, observation.parents);
|
|
583
|
+
const position = describePosition(node, within, observation.nodes);
|
|
585
584
|
return {
|
|
586
585
|
...(within === undefined ? {} : { within }),
|
|
587
586
|
...(position === undefined ? {} : { position }),
|
package/dist/agent/actions.d.ts
CHANGED
|
@@ -8,51 +8,52 @@
|
|
|
8
8
|
* the same descriptor projection. One owner for all three, so a recorded
|
|
9
9
|
* summary, a live event line, and a relocation candidate can never drift.
|
|
10
10
|
*/
|
|
11
|
-
import type {
|
|
11
|
+
import type { ViewportPoint, ViewportSize } from '../engine/surface.ts';
|
|
12
|
+
import { type NodeRedaction, type RedactedNode } from './observation.ts';
|
|
12
13
|
import { type NodeActionName, type PointActionName, type TraceTargetDescriptor, type TracePosition } from '../cache/trace.ts';
|
|
13
14
|
import type { ScrollDirection } from '../types.ts';
|
|
14
15
|
/** One committed grammar action, addressed by the node it actually ran against. */
|
|
15
16
|
export type RecordableAction = ({
|
|
16
17
|
readonly name: NodeActionName;
|
|
17
|
-
readonly node:
|
|
18
|
+
readonly node: RedactedNode;
|
|
18
19
|
} & Placement) | ({
|
|
19
20
|
readonly name: 'type';
|
|
20
|
-
readonly node:
|
|
21
|
+
readonly node: RedactedNode;
|
|
21
22
|
readonly value: string;
|
|
22
23
|
} & Placement) | ({
|
|
23
24
|
readonly name: 'typeSecret';
|
|
24
|
-
readonly node:
|
|
25
|
+
readonly node: RedactedNode;
|
|
25
26
|
readonly secret: string;
|
|
26
27
|
} & Placement) | ({
|
|
27
28
|
readonly name: 'press';
|
|
28
|
-
readonly node:
|
|
29
|
+
readonly node: RedactedNode;
|
|
29
30
|
readonly key: string;
|
|
30
31
|
} & Placement) | ({
|
|
31
32
|
readonly name: 'select';
|
|
32
|
-
readonly node:
|
|
33
|
+
readonly node: RedactedNode;
|
|
33
34
|
readonly value: string;
|
|
34
35
|
} & Placement) | ({
|
|
35
36
|
readonly name: 'check';
|
|
36
|
-
readonly node:
|
|
37
|
+
readonly node: RedactedNode;
|
|
37
38
|
readonly checked: boolean;
|
|
38
39
|
} & Placement)
|
|
39
40
|
/** Project-relative paths as the executor gave them; replay authorizes them again. */
|
|
40
41
|
| ({
|
|
41
42
|
readonly name: 'upload';
|
|
42
|
-
readonly node:
|
|
43
|
+
readonly node: RedactedNode;
|
|
43
44
|
readonly paths: readonly string[];
|
|
44
45
|
} & Placement)
|
|
45
46
|
/** The dragged node with its placement, and where it was dropped, with its own. */
|
|
46
47
|
| ({
|
|
47
48
|
readonly name: 'drag';
|
|
48
|
-
readonly node:
|
|
49
|
+
readonly node: RedactedNode;
|
|
49
50
|
readonly destination: {
|
|
50
|
-
readonly node:
|
|
51
|
+
readonly node: RedactedNode;
|
|
51
52
|
} & Placement;
|
|
52
53
|
} & Placement) | ({
|
|
53
54
|
readonly name: 'scroll';
|
|
54
55
|
readonly direction: ScrollDirection;
|
|
55
|
-
readonly node?:
|
|
56
|
+
readonly node?: RedactedNode;
|
|
56
57
|
/** The share of the viewport the node covered when scrolled, 0 to 1 (`ScrollAction.spans`). */
|
|
57
58
|
readonly spans?: number;
|
|
58
59
|
} & Placement)
|
|
@@ -61,7 +62,7 @@ export type RecordableAction = ({
|
|
|
61
62
|
readonly name: 'scrollUntil';
|
|
62
63
|
readonly text: string;
|
|
63
64
|
readonly direction: ScrollDirection;
|
|
64
|
-
readonly node?:
|
|
65
|
+
readonly node?: RedactedNode;
|
|
65
66
|
readonly spans?: number;
|
|
66
67
|
/** How many pages it took; in the summary, never in the trace, since a replay pages for itself. */
|
|
67
68
|
readonly screens: number;
|
|
@@ -92,7 +93,7 @@ export type RecordableAction = ({
|
|
|
92
93
|
readonly name: PointActionName;
|
|
93
94
|
readonly point: ViewportPoint;
|
|
94
95
|
readonly viewport: ViewportSize;
|
|
95
|
-
readonly under?:
|
|
96
|
+
readonly under?: RedactedNode;
|
|
96
97
|
};
|
|
97
98
|
/** Where the node sat when it was acted on: its container's key and its place among identical twins. */
|
|
98
99
|
export interface Placement {
|
|
@@ -100,14 +101,17 @@ export interface Placement {
|
|
|
100
101
|
readonly position?: TracePosition;
|
|
101
102
|
}
|
|
102
103
|
/**
|
|
103
|
-
* The key of the nearest named container a node sits in
|
|
104
|
-
* its row or list item
|
|
105
|
-
* Ten rows each with a "Delete" button are ten
|
|
106
|
-
* in the row that starts with Budget draft" is
|
|
104
|
+
* The key of the nearest named container a node sits in, the first text of
|
|
105
|
+
* its row or list item or else the container's own label, when that key says
|
|
106
|
+
* more than the node's own name. Ten rows each with a "Delete" button are ten
|
|
107
|
+
* identical descriptors; "Delete in the row that starts with Budget draft" is
|
|
108
|
+
* one. A row whose only text is its own (`<li>Alpha <button><svg/></li>`)
|
|
109
|
+
* is keyed by it. Only the nearest container counts: the list around a row
|
|
110
|
+
* with no text names every row alike, so its key would tell no twin apart.
|
|
107
111
|
*/
|
|
108
|
-
export declare function containerKey(id: string, nodes: ReadonlyMap<string,
|
|
112
|
+
export declare function containerKey(id: string, nodes: ReadonlyMap<string, RedactedNode>, parents: ReadonlyMap<string, string>): string | undefined;
|
|
109
113
|
/** Parent id of every non-root node, derived from the tree the node map indexes. */
|
|
110
|
-
export declare function parentsOf(nodes: ReadonlyMap<string,
|
|
114
|
+
export declare function parentsOf(nodes: ReadonlyMap<string, RedactedNode>): ReadonlyMap<string, string>;
|
|
111
115
|
/** How one committed action reads back: where it acted, and what it did. */
|
|
112
116
|
export interface DescribedAction {
|
|
113
117
|
/**
|
|
@@ -124,13 +128,18 @@ export interface DescribedAction {
|
|
|
124
128
|
/** Where a drag was dropped, described like its target; only a drag has one. */
|
|
125
129
|
readonly destination?: TraceTargetDescriptor;
|
|
126
130
|
}
|
|
127
|
-
/**
|
|
128
|
-
|
|
131
|
+
/**
|
|
132
|
+
* Describes one committed action for recording and for the live event. Its
|
|
133
|
+
* nodes pass the ledger again as it is now: a secret resolved after the
|
|
134
|
+
* capture (inside the action itself) masks what the capture still shows.
|
|
135
|
+
*/
|
|
136
|
+
export declare function describeAction(action: RecordableAction, redaction: NodeRedaction): DescribedAction;
|
|
129
137
|
/**
|
|
130
138
|
* Builds the durable descriptor for one resolved node: the semantic fields
|
|
131
139
|
* replay re-finds it by, plus the engine's structural selector hint (kept as
|
|
132
140
|
* provenance for tuned policies; the conservative relocator ignores it).
|
|
133
141
|
* Values a secure node holds are never part of it — descriptors carry how a
|
|
134
|
-
* node is named, not what it contains.
|
|
142
|
+
* node is named, not what it contains. The node was redacted with its
|
|
143
|
+
* observation (`redactNode`), each field in the one-line form read here too.
|
|
135
144
|
*/
|
|
136
|
-
export declare function describeTarget(node:
|
|
145
|
+
export declare function describeTarget(node: RedactedNode): TraceTargetDescriptor | undefined;
|
package/dist/agent/actions.js
CHANGED
|
@@ -8,32 +8,33 @@
|
|
|
8
8
|
* the same descriptor projection. One owner for all three, so a recorded
|
|
9
9
|
* summary, a live event line, and a relocation candidate can never drift.
|
|
10
10
|
*/
|
|
11
|
+
import { redactNode } from './observation.js';
|
|
11
12
|
import { bound, isNodeAction, MAX_TRACE_DESCRIPTOR_CHARS, MAX_TRACE_SUMMARY_CHARS, } from '../cache/trace.js';
|
|
12
13
|
import { sanitizeText } from '../internal/errors.js';
|
|
13
|
-
import {
|
|
14
|
+
import { collapseText } from '../internal/text.js';
|
|
14
15
|
/** Roles whose first text names the thing a control belongs to: a row's key, a list item's title. */
|
|
15
16
|
const CONTAINER_ROLES = new Set(['row', 'listitem', 'article', 'group', 'region', 'dialog', 'tabpanel']);
|
|
16
17
|
/** Longest container key kept. */
|
|
17
18
|
const MAX_WITHIN_CHARS = 80;
|
|
18
19
|
/**
|
|
19
|
-
* The key of the nearest named container a node sits in
|
|
20
|
-
* its row or list item
|
|
21
|
-
* Ten rows each with a "Delete" button are ten
|
|
22
|
-
* in the row that starts with Budget draft" is
|
|
20
|
+
* The key of the nearest named container a node sits in, the first text of
|
|
21
|
+
* its row or list item or else the container's own label, when that key says
|
|
22
|
+
* more than the node's own name. Ten rows each with a "Delete" button are ten
|
|
23
|
+
* identical descriptors; "Delete in the row that starts with Budget draft" is
|
|
24
|
+
* one. A row whose only text is its own (`<li>Alpha <button><svg/></li>`)
|
|
25
|
+
* is keyed by it. Only the nearest container counts: the list around a row
|
|
26
|
+
* with no text names every row alike, so its key would tell no twin apart.
|
|
23
27
|
*/
|
|
24
|
-
export function containerKey(id, nodes, parents
|
|
28
|
+
export function containerKey(id, nodes, parents) {
|
|
25
29
|
const node = nodes.get(id);
|
|
26
30
|
const own = node === undefined ? '' : squash(node.name ?? node.text ?? '');
|
|
27
31
|
let cursor = parents.get(id);
|
|
28
32
|
while (cursor !== undefined) {
|
|
29
33
|
const container = nodes.get(cursor);
|
|
30
34
|
if (container !== undefined && CONTAINER_ROLES.has(container.role ?? '')) {
|
|
31
|
-
const key = firstLeafText(container);
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
return clean === '' || squash(clean) === own ? undefined : clean;
|
|
35
|
-
}
|
|
36
|
-
return undefined;
|
|
35
|
+
const key = firstLeafText(container) ?? ownLabel(container);
|
|
36
|
+
const clean = key === undefined ? '' : bound(collapseText(key), MAX_WITHIN_CHARS);
|
|
37
|
+
return clean === '' || squash(clean) === own ? undefined : clean;
|
|
37
38
|
}
|
|
38
39
|
cursor = parents.get(cursor);
|
|
39
40
|
}
|
|
@@ -60,6 +61,11 @@ function firstLeafText(node) {
|
|
|
60
61
|
}
|
|
61
62
|
return undefined;
|
|
62
63
|
}
|
|
64
|
+
/** A container's own name or text, when it has one. */
|
|
65
|
+
function ownLabel(node) {
|
|
66
|
+
const label = (node.name ?? node.text ?? '').trim();
|
|
67
|
+
return label === '' ? undefined : label;
|
|
68
|
+
}
|
|
63
69
|
function squash(text) {
|
|
64
70
|
return text.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
65
71
|
}
|
|
@@ -74,16 +80,20 @@ const NODE_ACTION_PROSE = {
|
|
|
74
80
|
};
|
|
75
81
|
/** How each point verb reads: `tap the point (x, y)`, `hover over the point (x, y)`. */
|
|
76
82
|
const POINT_ACTION_PROSE = { tapAt: 'tap', hoverAt: 'hover over' };
|
|
77
|
-
/**
|
|
78
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Describes one committed action for recording and for the live event. Its
|
|
85
|
+
* nodes pass the ledger again as it is now: a secret resolved after the
|
|
86
|
+
* capture (inside the action itself) masks what the capture still shows.
|
|
87
|
+
*/
|
|
88
|
+
export function describeAction(action, redaction) {
|
|
79
89
|
const target = 'node' in action
|
|
80
|
-
? describePlaced(action.node, action,
|
|
90
|
+
? describePlaced(action.node, action, redaction)
|
|
81
91
|
: 'point' in action
|
|
82
|
-
? describePlaced(action.under, {},
|
|
92
|
+
? describePlaced(action.under, {}, redaction)
|
|
83
93
|
: undefined;
|
|
84
|
-
const destination = action.name === 'drag' ? describePlaced(action.destination.node, action.destination,
|
|
94
|
+
const destination = action.name === 'drag' ? describePlaced(action.destination.node, action.destination, redaction) : undefined;
|
|
85
95
|
const where = describeForSummary(target);
|
|
86
|
-
const safe = (value) => quote(redact(sanitizeText(value)));
|
|
96
|
+
const safe = (value) => quote(redaction.redact(sanitizeText(value)));
|
|
87
97
|
const prose = (() => {
|
|
88
98
|
if (isNodeAction(action))
|
|
89
99
|
return NODE_ACTION_PROSE[action.name](where);
|
|
@@ -134,13 +144,16 @@ export function describeAction(action, redact) {
|
|
|
134
144
|
};
|
|
135
145
|
}
|
|
136
146
|
/** A node's durable descriptor with the placement it was acted on in; undefined for no node or nothing durable. */
|
|
137
|
-
function describePlaced(node, placement,
|
|
138
|
-
|
|
147
|
+
function describePlaced(node, placement, redaction) {
|
|
148
|
+
if (node === undefined)
|
|
149
|
+
return undefined;
|
|
150
|
+
const { children: _children, ...fields } = node;
|
|
151
|
+
const described = describeTarget(redactNode(fields, redaction));
|
|
139
152
|
if (described === undefined)
|
|
140
153
|
return undefined;
|
|
141
154
|
return {
|
|
142
155
|
...described,
|
|
143
|
-
...(placement.within === undefined ? {} : { within: bound(placement.within, MAX_WITHIN_CHARS) }),
|
|
156
|
+
...(placement.within === undefined ? {} : { within: bound(redaction.redact(placement.within), MAX_WITHIN_CHARS) }),
|
|
144
157
|
...(placement.position === undefined ? {} : { position: placement.position }),
|
|
145
158
|
};
|
|
146
159
|
}
|
|
@@ -149,13 +162,14 @@ function describePlaced(node, placement, redact) {
|
|
|
149
162
|
* replay re-finds it by, plus the engine's structural selector hint (kept as
|
|
150
163
|
* provenance for tuned policies; the conservative relocator ignores it).
|
|
151
164
|
* Values a secure node holds are never part of it — descriptors carry how a
|
|
152
|
-
* node is named, not what it contains.
|
|
165
|
+
* node is named, not what it contains. The node was redacted with its
|
|
166
|
+
* observation (`redactNode`), each field in the one-line form read here too.
|
|
153
167
|
*/
|
|
154
|
-
export function describeTarget(node
|
|
168
|
+
export function describeTarget(node) {
|
|
155
169
|
const field = (value) => {
|
|
156
170
|
if (value === undefined)
|
|
157
171
|
return undefined;
|
|
158
|
-
const collapsed =
|
|
172
|
+
const collapsed = collapseText(value);
|
|
159
173
|
return collapsed === '' ? undefined : bound(collapsed, MAX_TRACE_DESCRIPTOR_CHARS);
|
|
160
174
|
};
|
|
161
175
|
const role = field(node.role);
|
package/dist/agent/model/sdk.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
import { withHint } from '../../internal/errors.js';
|
|
10
10
|
import { missingModelError } from '../../config/agent.js';
|
|
11
11
|
import { aiSdk, asSdkLanguageModel, loadAiSdk } from '../ai-sdk.js';
|
|
12
|
+
import { MODEL_REQUEST_HEADERS } from '../../internal/client-identity.js';
|
|
12
13
|
import { packageVersion } from '../../internal/package-version.js';
|
|
13
14
|
import { AgentError } from '../error.js';
|
|
14
15
|
import { isContextOverflow } from './overflow.js';
|
|
@@ -75,6 +76,7 @@ export function createModelAdapter(model, agentName = 'default') {
|
|
|
75
76
|
maxOutputTokens: call.maxOutputTokens,
|
|
76
77
|
...(providerOptions === undefined ? {} : { providerOptions: providerOptions }),
|
|
77
78
|
maxRetries: TRANSPORT_RETRIES,
|
|
79
|
+
headers: MODEL_REQUEST_HEADERS,
|
|
78
80
|
abortSignal: call.signal,
|
|
79
81
|
timeout: call.timeoutMs,
|
|
80
82
|
};
|
|
@@ -7,24 +7,23 @@
|
|
|
7
7
|
* executor's redacted view of a capture. Nothing else in the step observes
|
|
8
8
|
* the engine; the dispatcher and the pixel tier ask the feed.
|
|
9
9
|
*/
|
|
10
|
-
import type { SemanticNode } from '../engine/surface.ts';
|
|
11
10
|
import type { StepAgentDetails } from '../run/steps.ts';
|
|
12
11
|
import { AgentError } from './error.ts';
|
|
13
12
|
import type { ExecutorObservation, ExecutorObserveOptions, ExecutorTarget } from './executor.ts';
|
|
14
13
|
import type { AgentContext } from './invocation.ts';
|
|
15
|
-
import { type AgentObservation, type SemanticAgentObservation } from './observation.ts';
|
|
14
|
+
import { type AgentObservation, type RedactedNode, type SemanticAgentObservation } from './observation.ts';
|
|
16
15
|
import type { OperationQueue } from './operation-queue.ts';
|
|
17
16
|
import { type SettleMode } from './settle-policy.ts';
|
|
18
17
|
import type { StepAccounting } from './step-accounting.ts';
|
|
19
18
|
/** A target resolved to a node of the observation it was found in. */
|
|
20
19
|
export interface Resolved {
|
|
21
|
-
readonly node:
|
|
20
|
+
readonly node: RedactedNode;
|
|
22
21
|
readonly observation: SemanticAgentObservation;
|
|
23
22
|
}
|
|
24
23
|
/** What a target names on the newest screen: the id as the screen lists it, the node or nothing, and the screen looked at. */
|
|
25
24
|
export interface Lookup {
|
|
26
25
|
readonly id: string;
|
|
27
|
-
readonly node:
|
|
26
|
+
readonly node: RedactedNode | undefined;
|
|
28
27
|
readonly observation: SemanticAgentObservation;
|
|
29
28
|
}
|
|
30
29
|
export interface ObservationFeedOptions {
|
|
@@ -116,8 +115,8 @@ export declare class ObservationFeed {
|
|
|
116
115
|
* through the queued observe: this runs inside a queued action body, and a
|
|
117
116
|
* queued observation would wait on its own caller.
|
|
118
117
|
*/
|
|
119
|
-
relocate(stale:
|
|
120
|
-
node:
|
|
118
|
+
relocate(stale: RedactedNode): Promise<{
|
|
119
|
+
node: RedactedNode;
|
|
121
120
|
observation: SemanticAgentObservation;
|
|
122
121
|
} | undefined>;
|
|
123
122
|
/**
|
|
@@ -132,7 +132,7 @@ export class ObservationFeed {
|
|
|
132
132
|
...metadata,
|
|
133
133
|
text: observation.text,
|
|
134
134
|
truncated: observation.truncated,
|
|
135
|
-
...(options.tree === true ? { tree: projectTree(observation.tree, this.runtime.
|
|
135
|
+
...(options.tree === true ? { tree: projectTree(observation.tree, this.runtime.app.base?.origin) } : {}),
|
|
136
136
|
...pixels,
|
|
137
137
|
};
|
|
138
138
|
}
|
|
@@ -197,8 +197,7 @@ export class ObservationFeed {
|
|
|
197
197
|
* queued observation would wait on its own caller.
|
|
198
198
|
*/
|
|
199
199
|
async relocate(stale) {
|
|
200
|
-
const
|
|
201
|
-
const descriptor = describeTarget(stale, options.redact);
|
|
200
|
+
const descriptor = describeTarget(stale);
|
|
202
201
|
if (descriptor === undefined)
|
|
203
202
|
return undefined;
|
|
204
203
|
this.accounting.checkpoint();
|
|
@@ -206,7 +205,7 @@ export class ObservationFeed {
|
|
|
206
205
|
this.publish(observation);
|
|
207
206
|
if (observation.kind === 'pixels')
|
|
208
207
|
return undefined;
|
|
209
|
-
const relocated = relocateDescriptor(descriptor, observation.nodes
|
|
208
|
+
const relocated = relocateDescriptor(descriptor, observation.nodes);
|
|
210
209
|
if (relocated.kind !== 'found')
|
|
211
210
|
return undefined;
|
|
212
211
|
const node = observation.nodes.get(relocated.id);
|
|
@@ -376,11 +375,10 @@ export class ObservationFeed {
|
|
|
376
375
|
const earlier = this.lastSeen(id)?.node;
|
|
377
376
|
if (earlier === undefined)
|
|
378
377
|
return undefined;
|
|
379
|
-
const
|
|
380
|
-
const descriptor = describeTarget(earlier, options.redact);
|
|
378
|
+
const descriptor = describeTarget(earlier);
|
|
381
379
|
if (descriptor === undefined)
|
|
382
380
|
return undefined;
|
|
383
|
-
const relocated = relocateDescriptor(descriptor, latest.nodes
|
|
381
|
+
const relocated = relocateDescriptor(descriptor, latest.nodes);
|
|
384
382
|
return relocated.kind === 'found' ? latest.nodes.get(relocated.id) : undefined;
|
|
385
383
|
}
|
|
386
384
|
}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
/** Observation capture, redaction, and model serialization. */
|
|
2
2
|
import type { Observation, SemanticNode, ViewportSize } from '../engine/surface.ts';
|
|
3
|
+
import type { SecretLedger } from '../internal/redact.ts';
|
|
3
4
|
import type { VisionDegradation } from '../run/steps.ts';
|
|
4
5
|
import type { ExecutorNode, ExecutorObservation, ExecutorPixels } from './executor.ts';
|
|
5
6
|
/** Why pixels the caller asked for are not part of this observation. */
|
|
6
7
|
type PixelsWithheld = 'MASKING_UNPROVEN';
|
|
7
8
|
export type AgentObservation = AgentObservationMetadata & ({
|
|
8
9
|
readonly kind: 'semantic';
|
|
9
|
-
readonly nodes: ReadonlyMap<string,
|
|
10
|
+
readonly nodes: ReadonlyMap<string, RedactedNode>;
|
|
10
11
|
readonly parents: ReadonlyMap<string, string>;
|
|
11
|
-
readonly tree:
|
|
12
|
+
readonly tree: RedactedNode;
|
|
12
13
|
readonly truncated: boolean;
|
|
13
14
|
readonly pixels?: ExecutorPixels | undefined;
|
|
14
15
|
readonly pixelsWithheld?: PixelsWithheld | undefined;
|
|
@@ -37,10 +38,11 @@ interface AgentObservationMetadata {
|
|
|
37
38
|
*
|
|
38
39
|
* Pixels whose masking the engine cannot prove (fewer masked regions than
|
|
39
40
|
* secure nodes) are withheld before they reach a model or disk; the semantic
|
|
40
|
-
* tree never carries secure values.
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
41
|
+
* tree never carries secure values. Every string of every node is redacted
|
|
42
|
+
* once, here (`redactNode`): registered secret values are replaced by their
|
|
43
|
+
* stable secret name, and so is the leading part of one a field the engine
|
|
44
|
+
* cut at its limit ends with. A field holding one keeps no selection, as a
|
|
45
|
+
* secure field keeps none.
|
|
44
46
|
*/
|
|
45
47
|
export declare function prepareObservation(observation: Observation, options: {
|
|
46
48
|
redact: (text: string) => string;
|
|
@@ -52,6 +54,34 @@ export declare function prepareObservation(observation: Observation, options: {
|
|
|
52
54
|
/** The app base URL's origin: link targets on it render as paths, which navigation resolves against it. */
|
|
53
55
|
appOrigin?: string | undefined;
|
|
54
56
|
}): AgentObservation;
|
|
57
|
+
/** What redacts a node: whole values, and the leading part of one a cut field ends with. */
|
|
58
|
+
export type NodeRedaction = Pick<SecretLedger, 'redact' | 'redactCut'>;
|
|
59
|
+
declare const REDACTED: unique symbol;
|
|
60
|
+
/**
|
|
61
|
+
* A semantic node every string of which has passed the attempt's secret
|
|
62
|
+
* ledger (`redactNode`). Model-facing text and executor trees render only
|
|
63
|
+
* from this shape, so a node no redaction touched cannot reach them.
|
|
64
|
+
*/
|
|
65
|
+
export interface RedactedNode extends Omit<SemanticNode, 'children'> {
|
|
66
|
+
readonly [REDACTED]: true;
|
|
67
|
+
readonly children?: readonly RedactedNode[];
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The node and its subtree with every secret taken out of every field
|
|
71
|
+
* (`NODE_FIELDS`), before any consumer reads it: the model's text, the
|
|
72
|
+
* executor's tree, cache descriptors and anchors, failure evidence.
|
|
73
|
+
*/
|
|
74
|
+
export declare function redactNode(node: SemanticNode, redaction: NodeRedaction): RedactedNode;
|
|
75
|
+
/**
|
|
76
|
+
* An earlier capture's nodes redacted again through the ledger as it is now,
|
|
77
|
+
* each on its own fields, then linked to its redacted-again children, so the
|
|
78
|
+
* tree keeps its shape (which nodes are leaves) without a subtree redacted
|
|
79
|
+
* once per ancestor. A secret a provider resolved after the capture masks
|
|
80
|
+
* text the capture still shows, so the two captures read alike wherever the
|
|
81
|
+
* screen did not change. Redacting redacted text changes nothing else: a
|
|
82
|
+
* marker is never read again.
|
|
83
|
+
*/
|
|
84
|
+
export declare function redactNodesAgain(nodes: ReadonlyMap<string, RedactedNode>, redaction: NodeRedaction): ReadonlyMap<string, RedactedNode>;
|
|
55
85
|
/** Phase metrics describe a node count only when the capture actually read nodes. */
|
|
56
86
|
export declare function observationDetail(observation: AgentObservation): {
|
|
57
87
|
count?: number;
|
|
@@ -82,21 +112,20 @@ export declare const INTERACTIVE_ROLES: ReadonlySet<string>;
|
|
|
82
112
|
*/
|
|
83
113
|
export declare function interactiveNodeCount(observation: Pick<ExecutorObservation, 'text'>): number;
|
|
84
114
|
/**
|
|
85
|
-
* Projects
|
|
86
|
-
*
|
|
87
|
-
* selection at all for a secure node, and a link target bounded as a line
|
|
115
|
+
* Projects a redacted tree onto the executor-facing node shape, the fields
|
|
116
|
+
* the text serialization renders, with a link target bounded as a line
|
|
88
117
|
* bounds it: a target on `appOrigin` keeps that origin and loses no more of
|
|
89
118
|
* its path than the line does. Selectors stay behind — they are relocation
|
|
90
119
|
* material for the trace cache, not something a brain reasons about.
|
|
91
120
|
*/
|
|
92
|
-
export declare function projectTree(node:
|
|
121
|
+
export declare function projectTree(node: RedactedNode, appOrigin?: string): ExecutorNode;
|
|
93
122
|
/**
|
|
94
123
|
* Renders one node as `#id role "name" text="..." value="..." selection="..." [states]`. Role-less text
|
|
95
124
|
* holders omit the role token entirely: on a large screen they are half the
|
|
96
125
|
* lines, and the model needs their text, not a filler word. A link target on
|
|
97
126
|
* `appOrigin`, the app base URL's, renders as its path.
|
|
98
127
|
*/
|
|
99
|
-
export declare function formatNode(node:
|
|
128
|
+
export declare function formatNode(node: RedactedNode, depth: number, appOrigin?: string): string;
|
|
100
129
|
/**
|
|
101
130
|
* What the screen looks like, independent of which observation looked at it.
|
|
102
131
|
*
|