@roll-agent/browser-use-agent 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +36 -8
- package/dist/browser-observation.d.ts +1 -0
- package/dist/execution-visual-feedback.d.ts +61 -0
- package/dist/goal/compact-request.d.ts +5 -0
- package/dist/goal/contracts.d.ts +1732 -0
- package/dist/goal/decisions.d.ts +55 -0
- package/dist/goal/dependency-observation.d.ts +7 -0
- package/dist/goal/execution-context.d.ts +593 -0
- package/dist/goal/form-context.d.ts +639 -0
- package/dist/goal/form-handoff.d.ts +152 -0
- package/dist/goal/form-loop-progress.d.ts +9 -0
- package/dist/goal/form-policy.d.ts +13 -0
- package/dist/goal/host.d.ts +6 -0
- package/dist/goal/interaction-feedback.d.ts +82 -0
- package/dist/goal/loop.d.ts +12 -0
- package/dist/goal/observation-retry.d.ts +3 -0
- package/dist/goal/observation.d.ts +981 -0
- package/dist/goal/operate-handoff.d.ts +3 -0
- package/dist/goal/picker-observation.d.ts +56 -0
- package/dist/goal/picker-state.d.ts +96 -0
- package/dist/goal/reachability.d.ts +2 -0
- package/dist/goal/task-evidence.d.ts +33 -0
- package/dist/goal/task-freshness.d.ts +86 -0
- package/dist/goal/task-loop.d.ts +5 -0
- package/dist/goal/task-observation.d.ts +5 -0
- package/dist/goal/task-policy.d.ts +174 -0
- package/dist/goal/task-progress-policy.d.ts +25 -0
- package/dist/goal/task-progress.d.ts +353 -0
- package/dist/goal/task-sources.d.ts +13 -0
- package/dist/index.js +1 -1
- package/dist/native-visual-activity-session.d.ts +38 -1
- package/dist/tools/browser-operate.d.ts +252 -0
- package/dist/tools/zhipin-diagnose-browser-state.d.ts +4 -4
- package/dist/tools/zhipin-generate-reply-preview.d.ts +1 -1
- package/dist/tools/zhipin-get-candidate-info.d.ts +1 -1
- package/dist/tools/zhipin-scroll-view.d.ts +5 -5
- package/package.json +3 -3
- package/references/browser-exploration.md +52 -0
- package/references/env.yaml +11 -0
|
@@ -1,16 +1,53 @@
|
|
|
1
1
|
import type { VisualActivityTone } from "./visual-activity.ts";
|
|
2
2
|
import type { NativeMouseClickPreview, NativeMouseMotionObserver, NativeMouseMotionPreview } from "./native-mouse-motion.ts";
|
|
3
3
|
type NativeVisualTarget = {
|
|
4
|
-
evaluateJson<T = unknown>(expression: string
|
|
4
|
+
evaluateJson<T = unknown>(expression: string, options?: {
|
|
5
|
+
timeoutMs?: number;
|
|
6
|
+
}): Promise<T>;
|
|
5
7
|
};
|
|
6
8
|
type NativeVisualHighlightOptions = {
|
|
7
9
|
readonly label?: string;
|
|
8
10
|
readonly padding?: number;
|
|
9
11
|
readonly tone?: VisualActivityTone;
|
|
10
12
|
};
|
|
13
|
+
export type NativeExecutionCard = {
|
|
14
|
+
readonly ownerId: string;
|
|
15
|
+
readonly epoch: number;
|
|
16
|
+
readonly revision: number;
|
|
17
|
+
readonly actionRevision: number;
|
|
18
|
+
readonly title: string;
|
|
19
|
+
readonly stage: string;
|
|
20
|
+
readonly action?: string;
|
|
21
|
+
readonly target?: string;
|
|
22
|
+
readonly recent: readonly string[];
|
|
23
|
+
readonly tone?: VisualActivityTone;
|
|
24
|
+
readonly lingerMs?: number;
|
|
25
|
+
};
|
|
26
|
+
export type NativeExecutionRect = {
|
|
27
|
+
readonly ownerId: string;
|
|
28
|
+
readonly epoch: number;
|
|
29
|
+
readonly actionRevision: number;
|
|
30
|
+
readonly x: number;
|
|
31
|
+
readonly y: number;
|
|
32
|
+
readonly width: number;
|
|
33
|
+
readonly height: number;
|
|
34
|
+
};
|
|
35
|
+
export type NativeExecutionPointer = {
|
|
36
|
+
readonly ownerId: string;
|
|
37
|
+
readonly epoch: number;
|
|
38
|
+
readonly actionRevision: number;
|
|
39
|
+
readonly type: "mouseMoved" | "mousePressed";
|
|
40
|
+
readonly x: number;
|
|
41
|
+
readonly y: number;
|
|
42
|
+
};
|
|
11
43
|
export declare class NativeVisualActivitySession implements NativeMouseMotionObserver {
|
|
12
44
|
private readonly target;
|
|
45
|
+
private executionLifecycleStarted;
|
|
13
46
|
constructor(target: NativeVisualTarget);
|
|
47
|
+
showExecutionCard(card: NativeExecutionCard, mode?: "begin" | "update" | "complete"): Promise<boolean>;
|
|
48
|
+
clearExecutionCard(card: Pick<NativeExecutionCard, "ownerId" | "epoch" | "revision" | "actionRevision">): Promise<boolean>;
|
|
49
|
+
highlightExecutionRect(rect: NativeExecutionRect): Promise<boolean>;
|
|
50
|
+
previewExecutionPointer(pointer: NativeExecutionPointer): Promise<boolean>;
|
|
14
51
|
begin(label: string, tone?: VisualActivityTone): Promise<boolean>;
|
|
15
52
|
highlightSelector(selector: string, options?: NativeVisualHighlightOptions): Promise<boolean>;
|
|
16
53
|
previewMouseMotion(preview: NativeMouseMotionPreview): Promise<void>;
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
export declare const browserOperate: import("@roll-agent/sdk").ToolDefinition<{
|
|
2
|
+
pageId: string;
|
|
3
|
+
allowedOrigins: string[];
|
|
4
|
+
goal: string;
|
|
5
|
+
values?: {
|
|
6
|
+
name: string;
|
|
7
|
+
text: string;
|
|
8
|
+
}[] | undefined;
|
|
9
|
+
timeoutMs?: number | undefined;
|
|
10
|
+
model?: string | undefined;
|
|
11
|
+
browserActionApproval?: {
|
|
12
|
+
id: string;
|
|
13
|
+
} | undefined;
|
|
14
|
+
formTask?: {
|
|
15
|
+
mode: "create" | "edit";
|
|
16
|
+
fields: ({
|
|
17
|
+
name: string;
|
|
18
|
+
intent: "set";
|
|
19
|
+
valueName: string;
|
|
20
|
+
comparison?: "auto" | "literal" | "semantic" | undefined;
|
|
21
|
+
} | {
|
|
22
|
+
name: string;
|
|
23
|
+
intent: "preserve";
|
|
24
|
+
})[];
|
|
25
|
+
stopAt?: "applied" | "current-view" | undefined;
|
|
26
|
+
} | undefined;
|
|
27
|
+
readTask?: {
|
|
28
|
+
target: string;
|
|
29
|
+
captureView: string;
|
|
30
|
+
outputs: string[];
|
|
31
|
+
terminal: {
|
|
32
|
+
view: string;
|
|
33
|
+
selectedTab?: string | undefined;
|
|
34
|
+
};
|
|
35
|
+
} | undefined;
|
|
36
|
+
delegatedDecisions?: string[] | undefined;
|
|
37
|
+
maxTextCalls?: number | undefined;
|
|
38
|
+
maxRecoveryDecisions?: number | undefined;
|
|
39
|
+
blockedNames?: string[] | undefined;
|
|
40
|
+
engine?: "jev" | "sampling" | undefined;
|
|
41
|
+
strategy?: "task" | "fields" | undefined;
|
|
42
|
+
maxSteps?: number | undefined;
|
|
43
|
+
}, {
|
|
44
|
+
status: "failed" | "cancelled" | "needs_input" | "model_done" | "interaction_done" | "needs_reasoning" | "blocked" | "step_limit";
|
|
45
|
+
elapsedMs: number;
|
|
46
|
+
verified: false;
|
|
47
|
+
steps: {
|
|
48
|
+
executed: boolean;
|
|
49
|
+
step: number;
|
|
50
|
+
provider: string;
|
|
51
|
+
requestedModel: string;
|
|
52
|
+
resolvedModel: string;
|
|
53
|
+
operation: string;
|
|
54
|
+
observationMs: number;
|
|
55
|
+
decisionMs: number;
|
|
56
|
+
actionMs: number;
|
|
57
|
+
target?: string | undefined;
|
|
58
|
+
error?: string | undefined;
|
|
59
|
+
usage?: {
|
|
60
|
+
cost?: number | undefined;
|
|
61
|
+
inputTokens?: number | undefined;
|
|
62
|
+
outputTokens?: number | undefined;
|
|
63
|
+
} | undefined;
|
|
64
|
+
distributions?: Record<string, {
|
|
65
|
+
probabilities: Record<string, number>;
|
|
66
|
+
confidence?: number | undefined;
|
|
67
|
+
}> | undefined;
|
|
68
|
+
valueName?: string | undefined;
|
|
69
|
+
decisionAttempts?: number | undefined;
|
|
70
|
+
textMs?: number | undefined;
|
|
71
|
+
recovery?: boolean | undefined;
|
|
72
|
+
requirement?: string | undefined;
|
|
73
|
+
}[];
|
|
74
|
+
error?: string | undefined;
|
|
75
|
+
progress?: {
|
|
76
|
+
missing: string[];
|
|
77
|
+
taskRunId: string;
|
|
78
|
+
contractDigest: string;
|
|
79
|
+
phase: "collect" | "return" | "done";
|
|
80
|
+
evidence: {
|
|
81
|
+
id: string;
|
|
82
|
+
url: string;
|
|
83
|
+
frameId: string;
|
|
84
|
+
documentId: string;
|
|
85
|
+
text: string;
|
|
86
|
+
field: string;
|
|
87
|
+
end: number;
|
|
88
|
+
observedAt: string;
|
|
89
|
+
observationId: string;
|
|
90
|
+
regionId: string;
|
|
91
|
+
regionName: string;
|
|
92
|
+
start: number;
|
|
93
|
+
attribution: "model-selected-observed-text";
|
|
94
|
+
}[];
|
|
95
|
+
invalidatedEvidence: {
|
|
96
|
+
id: string;
|
|
97
|
+
url: string;
|
|
98
|
+
frameId: string;
|
|
99
|
+
documentId: string;
|
|
100
|
+
text: string;
|
|
101
|
+
field: string;
|
|
102
|
+
end: number;
|
|
103
|
+
observedAt: string;
|
|
104
|
+
observationId: string;
|
|
105
|
+
regionId: string;
|
|
106
|
+
regionName: string;
|
|
107
|
+
start: number;
|
|
108
|
+
attribution: "model-selected-observed-text";
|
|
109
|
+
}[];
|
|
110
|
+
conflicts: number;
|
|
111
|
+
capturedRegion?: {
|
|
112
|
+
id: string;
|
|
113
|
+
url: string;
|
|
114
|
+
name: string;
|
|
115
|
+
frameId: string;
|
|
116
|
+
documentId: string;
|
|
117
|
+
digest: string;
|
|
118
|
+
} | undefined;
|
|
119
|
+
} | undefined;
|
|
120
|
+
execution?: {
|
|
121
|
+
changes: {
|
|
122
|
+
scope: "delegated" | "preserved" | "unassigned";
|
|
123
|
+
kind: "changed" | "became_unknown" | "appeared";
|
|
124
|
+
field: string;
|
|
125
|
+
observationId: string;
|
|
126
|
+
identity: string;
|
|
127
|
+
before?: string | undefined;
|
|
128
|
+
after?: string | undefined;
|
|
129
|
+
afterAction?: string | undefined;
|
|
130
|
+
}[];
|
|
131
|
+
revision: number;
|
|
132
|
+
invocationId: string;
|
|
133
|
+
documentChanged: boolean;
|
|
134
|
+
form?: {
|
|
135
|
+
mode: "create" | "edit";
|
|
136
|
+
stopAt: "applied" | "current-view";
|
|
137
|
+
fields: {
|
|
138
|
+
status: "unknown" | "satisfied" | "unsatisfied";
|
|
139
|
+
id: string;
|
|
140
|
+
name: string;
|
|
141
|
+
reason: string;
|
|
142
|
+
intent: "set" | "preserve";
|
|
143
|
+
comparison: "auto" | "literal" | "semantic";
|
|
144
|
+
expected?: string | undefined;
|
|
145
|
+
target?: {
|
|
146
|
+
label: string;
|
|
147
|
+
key: string;
|
|
148
|
+
optionsKey: string;
|
|
149
|
+
} | undefined;
|
|
150
|
+
fingerprint?: string | undefined;
|
|
151
|
+
binding?: {
|
|
152
|
+
role: string;
|
|
153
|
+
name: string;
|
|
154
|
+
identity: string;
|
|
155
|
+
key?: string | undefined;
|
|
156
|
+
fieldLabel?: string | undefined;
|
|
157
|
+
readOnlyDisplay?: true | undefined;
|
|
158
|
+
} | undefined;
|
|
159
|
+
entry?: {
|
|
160
|
+
role: string;
|
|
161
|
+
name: string;
|
|
162
|
+
identity: string;
|
|
163
|
+
key?: string | undefined;
|
|
164
|
+
fieldLabel?: string | undefined;
|
|
165
|
+
readOnlyDisplay?: true | undefined;
|
|
166
|
+
} | undefined;
|
|
167
|
+
entryProof?: {
|
|
168
|
+
status: "unknown" | "satisfied" | "unsatisfied";
|
|
169
|
+
fingerprint: string;
|
|
170
|
+
} | undefined;
|
|
171
|
+
initial?: string | undefined;
|
|
172
|
+
current?: string | undefined;
|
|
173
|
+
semanticProof?: {
|
|
174
|
+
status: "satisfied" | "unsatisfied";
|
|
175
|
+
fingerprint: string;
|
|
176
|
+
} | undefined;
|
|
177
|
+
}[];
|
|
178
|
+
panel?: {
|
|
179
|
+
status: "pending" | "rejected" | "accepted";
|
|
180
|
+
key: string;
|
|
181
|
+
queryIdentity?: string | undefined;
|
|
182
|
+
} | undefined;
|
|
183
|
+
focus?: string | undefined;
|
|
184
|
+
} | undefined;
|
|
185
|
+
focus?: string | undefined;
|
|
186
|
+
latestObservation?: string | undefined;
|
|
187
|
+
lastAction?: {
|
|
188
|
+
executed: boolean;
|
|
189
|
+
operation: string;
|
|
190
|
+
target?: string | undefined;
|
|
191
|
+
error?: string | undefined;
|
|
192
|
+
} | undefined;
|
|
193
|
+
} | undefined;
|
|
194
|
+
resolvedValues?: {
|
|
195
|
+
source: "selected" | "supplied" | "derived" | "generated";
|
|
196
|
+
text: string;
|
|
197
|
+
field: string;
|
|
198
|
+
evidence: string;
|
|
199
|
+
sourceIds: string[];
|
|
200
|
+
targetKey?: string | undefined;
|
|
201
|
+
}[] | undefined;
|
|
202
|
+
question?: string | undefined;
|
|
203
|
+
handoff?: {
|
|
204
|
+
fields: {
|
|
205
|
+
status: "unknown" | "satisfied" | "unsatisfied";
|
|
206
|
+
id: string;
|
|
207
|
+
name: string;
|
|
208
|
+
reason: string;
|
|
209
|
+
intent: "set" | "preserve";
|
|
210
|
+
valueTruncated: boolean;
|
|
211
|
+
expected?: string | undefined;
|
|
212
|
+
current?: string | undefined;
|
|
213
|
+
}[];
|
|
214
|
+
observationFresh: boolean;
|
|
215
|
+
diagnosticsOmitted: true;
|
|
216
|
+
totalSteps: number;
|
|
217
|
+
omittedSteps: number;
|
|
218
|
+
next: string;
|
|
219
|
+
} | undefined;
|
|
220
|
+
recoveryDecisions?: number | undefined;
|
|
221
|
+
fieldEvidence?: {
|
|
222
|
+
status: "pending" | "preserved" | "applied";
|
|
223
|
+
field: string;
|
|
224
|
+
instruction: string;
|
|
225
|
+
targetKey: string;
|
|
226
|
+
expected?: string | undefined;
|
|
227
|
+
source?: string | undefined;
|
|
228
|
+
current?: string | undefined;
|
|
229
|
+
}[] | undefined;
|
|
230
|
+
pendingRequirements?: string[] | undefined;
|
|
231
|
+
finalObservation?: unknown;
|
|
232
|
+
textCalls?: {
|
|
233
|
+
elapsedMs: number;
|
|
234
|
+
purpose: "preflight" | "resolve_value" | "resolve_batch" | "validate_value" | "validate_batch" | "review_field" | "review" | "clarify";
|
|
235
|
+
cached: boolean;
|
|
236
|
+
startedAtMs?: number | undefined;
|
|
237
|
+
endedAtMs?: number | undefined;
|
|
238
|
+
modelCalls?: {
|
|
239
|
+
elapsedMs: number;
|
|
240
|
+
provider: string;
|
|
241
|
+
requestedModel: string;
|
|
242
|
+
resolvedModel: string;
|
|
243
|
+
error?: string | undefined;
|
|
244
|
+
usage?: {
|
|
245
|
+
cost?: number | undefined;
|
|
246
|
+
inputTokens?: number | undefined;
|
|
247
|
+
outputTokens?: number | undefined;
|
|
248
|
+
} | undefined;
|
|
249
|
+
fallback?: boolean | undefined;
|
|
250
|
+
}[] | undefined;
|
|
251
|
+
}[] | undefined;
|
|
252
|
+
}>;
|
|
@@ -14,8 +14,8 @@ declare const StorageEntrySummarySchema: z.ZodObject<{
|
|
|
14
14
|
}, "strip", z.ZodTypeAny, {
|
|
15
15
|
key: string;
|
|
16
16
|
area: "localStorage" | "sessionStorage";
|
|
17
|
-
valueLength: number;
|
|
18
17
|
valueKind: "string" | "json" | "empty";
|
|
18
|
+
valueLength: number;
|
|
19
19
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
20
20
|
jsonTopLevelKeys?: string[] | undefined;
|
|
21
21
|
jsonArrayLength?: number | undefined;
|
|
@@ -25,8 +25,8 @@ declare const StorageEntrySummarySchema: z.ZodObject<{
|
|
|
25
25
|
}, {
|
|
26
26
|
key: string;
|
|
27
27
|
area: "localStorage" | "sessionStorage";
|
|
28
|
-
valueLength: number;
|
|
29
28
|
valueKind: "string" | "json" | "empty";
|
|
29
|
+
valueLength: number;
|
|
30
30
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
31
31
|
jsonTopLevelKeys?: string[] | undefined;
|
|
32
32
|
jsonArrayLength?: number | undefined;
|
|
@@ -303,8 +303,8 @@ export declare const zhipinDiagnoseBrowserState: import("@roll-agent/sdk").ToolD
|
|
|
303
303
|
localStorage: {
|
|
304
304
|
key: string;
|
|
305
305
|
area: "localStorage" | "sessionStorage";
|
|
306
|
-
valueLength: number;
|
|
307
306
|
valueKind: "string" | "json" | "empty";
|
|
307
|
+
valueLength: number;
|
|
308
308
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
309
309
|
jsonTopLevelKeys?: string[] | undefined;
|
|
310
310
|
jsonArrayLength?: number | undefined;
|
|
@@ -315,8 +315,8 @@ export declare const zhipinDiagnoseBrowserState: import("@roll-agent/sdk").ToolD
|
|
|
315
315
|
sessionStorage: {
|
|
316
316
|
key: string;
|
|
317
317
|
area: "localStorage" | "sessionStorage";
|
|
318
|
-
valueLength: number;
|
|
319
318
|
valueKind: "string" | "json" | "empty";
|
|
319
|
+
valueLength: number;
|
|
320
320
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
321
321
|
jsonTopLevelKeys?: string[] | undefined;
|
|
322
322
|
jsonArrayLength?: number | undefined;
|
|
@@ -30,11 +30,11 @@ export declare const zhipinGenerateReplyPreview: import("@roll-agent/sdk").ToolD
|
|
|
30
30
|
success: boolean;
|
|
31
31
|
error?: string | undefined;
|
|
32
32
|
elapsedMs?: number | undefined;
|
|
33
|
+
stage?: string | undefined;
|
|
33
34
|
confidence?: number | undefined;
|
|
34
35
|
requestId?: string | undefined;
|
|
35
36
|
expiresAt?: number | undefined;
|
|
36
37
|
suggestedReply?: string | undefined;
|
|
37
|
-
stage?: string | undefined;
|
|
38
38
|
preparedReplyId?: string | undefined;
|
|
39
39
|
clientTimeoutMs?: number | undefined;
|
|
40
40
|
lastStartedPhase?: string | undefined;
|
|
@@ -40,8 +40,8 @@ export declare const zhipinGetCandidateInfo: import("@roll-agent/sdk").ToolDefin
|
|
|
40
40
|
source: "candidate_message" | "conversation_history" | "candidate_expected_location" | "communication_position";
|
|
41
41
|
text: string;
|
|
42
42
|
confidence: number;
|
|
43
|
-
city?: string | undefined;
|
|
44
43
|
intent?: "nearby_store" | "store_address" | "expected_area" | undefined;
|
|
44
|
+
city?: string | undefined;
|
|
45
45
|
}[];
|
|
46
46
|
chatMessages: {
|
|
47
47
|
index: number;
|
|
@@ -9,15 +9,13 @@ type ZhipinScrollViewDeps = {
|
|
|
9
9
|
export declare function setZhipinScrollViewDepsForTests(override: Partial<ZhipinScrollViewDeps> | undefined): void;
|
|
10
10
|
export declare const zhipinScrollView: import("@roll-agent/sdk").ToolDefinition<{
|
|
11
11
|
surface: "chat-list" | "chat-history" | "recommend-list";
|
|
12
|
+
steps?: number | undefined;
|
|
12
13
|
direction?: "up" | "down" | undefined;
|
|
13
14
|
distance?: number | undefined;
|
|
14
|
-
steps?: number | undefined;
|
|
15
15
|
settleMs?: number | undefined;
|
|
16
16
|
}, {
|
|
17
17
|
success: boolean;
|
|
18
|
-
|
|
19
|
-
stepsRequested: number;
|
|
20
|
-
stepsCompleted: number;
|
|
18
|
+
position: "unknown" | "middle" | "top" | "bottom" | "only-page";
|
|
21
19
|
before: {
|
|
22
20
|
containerFound: boolean;
|
|
23
21
|
containerLabel: string;
|
|
@@ -38,8 +36,10 @@ export declare const zhipinScrollView: import("@roll-agent/sdk").ToolDefinition<
|
|
|
38
36
|
atEnd: boolean;
|
|
39
37
|
itemCount: number;
|
|
40
38
|
};
|
|
39
|
+
direction: "up" | "down";
|
|
40
|
+
stepsRequested: number;
|
|
41
|
+
stepsCompleted: number;
|
|
41
42
|
reachedBoundary: boolean;
|
|
42
|
-
position: "unknown" | "middle" | "top" | "bottom" | "only-page";
|
|
43
43
|
surface: "chat-list" | "chat-history" | "recommend-list";
|
|
44
44
|
atTop: boolean;
|
|
45
45
|
atBottom: boolean;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@roll-agent/browser-use-agent",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.27.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -44,9 +44,9 @@
|
|
|
44
44
|
}
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
+
"@roll-agent/browser": "0.12.0",
|
|
47
48
|
"@roll-agent/reply-authority-client": "0.7.0",
|
|
48
|
-
"@roll-agent/sdk": "0.
|
|
49
|
-
"@roll-agent/browser": "0.11.0",
|
|
49
|
+
"@roll-agent/sdk": "0.7.0",
|
|
50
50
|
"zod": "^3.25.76"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
@@ -4,6 +4,48 @@
|
|
|
4
4
|
|
|
5
5
|
单步 `browser_snapshot` / `click_ref` / `type_ref` 的区域观察、上下文选择和严格 ref 用法见 [通用 ref 操作](generic-browser-refs.md)。
|
|
6
6
|
|
|
7
|
+
遵循用户明确指定的工具;未知目标先观察。资料与文案齐全、字段范围和停止点明确的多字段表单,尤其涉及下拉、联动字段或弹层编辑器时,首选 `browser_operate`;无需提前知道点击顺序,不因宿主能拆步骤就逐字段执行。已确定步骤的短脚本、批量输入与断言用 `browser_execute`,它不调用 Jev。单次点击/输入、范围尚不明确的探索或委派受阻后的局部修正用单步工具。两者都不能绕过发布/保存授权、动态观察、结果验证或失败后现场检查。
|
|
8
|
+
|
|
9
|
+
## 通过完整目标执行
|
|
10
|
+
|
|
11
|
+
`browser_operate` 默认 `strategy:"task"`。调用方交代原始目标、已知事实和已准备文案;所选决策引擎在当前页面选择下一步。例:
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{
|
|
15
|
+
"pageId": "真实页面 ID",
|
|
16
|
+
"goal": "填写岗位草稿。优先处理前置选择和必填字段,使用资料中的内容,停止在发布前。",
|
|
17
|
+
"values": [
|
|
18
|
+
{ "name": "职位名称", "text": "餐饮兼职服务员" },
|
|
19
|
+
{ "name": "岗位说明", "text": "调用方已准备并允许使用的完整岗位文案" },
|
|
20
|
+
{ "name": "最低月薪", "text": "5000" },
|
|
21
|
+
{ "name": "最高月薪", "text": "6000" }
|
|
22
|
+
],
|
|
23
|
+
"strategy": "task",
|
|
24
|
+
"allowedOrigins": ["https://example.com"],
|
|
25
|
+
"blockedNames": ["发布", "提交"]
|
|
26
|
+
}
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
示例中的页面、origin 和事实必须替换为实际输入。必填标记不允许编造缺失事实;用户要求的选填字段仍须处理。调用方需要拟写内容时在交付前准备好;决策引擎不负责自由写作。操作过程中无需宿主逐项检查、字段完成复核或内部恢复。
|
|
30
|
+
|
|
31
|
+
一次决策请求共享当前页面状态,询问一个带目标的完整动作,以及各输入框对应的来源值。完整动作把操作与目标绑定,避免不兼容组合。各来源问题按对应输入框独立执行,只消费选中动作的来源答案;输入框与其来源问题显式绑定,未使用分支的置信度不影响执行。决策引擎可优先处理前置选择和必填字段;编辑入口先点击,再观察里面的真实字段,不能强制给按钮解析一个输入值。
|
|
32
|
+
|
|
33
|
+
决策输入中的 `pickerFields` 单独表示选择控件,元素通过字段 ID 关联触发器或候选项。关系来源区分 ARIA 和唯一局部 DOM 包含关系;无法唯一定位的归属保持未知,不按距离猜测。原生标签、placeholder、空触发器展示文字和先前观察到的标签分别标注来源;只有文档、frame 和实际节点身份一致时才保留旧标签。节点替换后不沿用。
|
|
34
|
+
|
|
35
|
+
`valueKind` 区分输入值、已应用值、查询词、占位内容、候选文字和纯展示文字。`sourceValueMatches`/`writtenValueMatches` 仅是明确范围的字符串相等线索,不表示任务完成;候选菜单中的文字不能充当字段值。下拉展开状态 `expanded` 与菜单当前是否可见 `panelVisible` 分开表达,异步菜单未出现不等于控件已关闭。隐藏 backing input 只用于提供有值/空值证据,不向模型输出其内部编码。
|
|
36
|
+
|
|
37
|
+
动作描述明确区分“打开某字段”“选择该字段的候选”“向搜索框输入查询”。完成问题要求对照每个请求字段的实际结果,不能因为页面某处出现了目标文字就结束。这些输入同时适用于默认的 Sampling 与显式启用的 Jev 决策引擎,不增加宿主复核或置信度门禁。
|
|
38
|
+
|
|
39
|
+
输入候选由代码从完整 `values` 和有限的 goal 原文片段中提供,决策引擎选择后代码复制完整原文,不把预览截断内容当成实际值。保留无匹配选项;原文候选覆盖不全、需要转换或没有所需文案时返回 `needs_input`,调用方提供针对该控件的值后继续。原生 select 只使用观察到的 option;自定义下拉通过点击、查询和选择推进。页面文本是 UI 证据,不能冒充用户数据。
|
|
40
|
+
|
|
41
|
+
代码保留站点/策略限制、文档与控件身份检查、可操作性检查、输入精确读回、取消与停滞预算。不执行模型生成的脚本/选择器,不在不确定的动作后自动重放。搜索文字不等于已应用的字段值,模型根据每轮新观察继续选择和确认。
|
|
42
|
+
|
|
43
|
+
默认 `sampling` 使用 Roll MCP Sampling 模型;core-managed Agent 通过 Roll 配置 `browser.operate.engine: jev` 切换到 TypeSafe,外部管理的 Agent 由进程管理方设置 `BROWSER_OPERATE_ENGINE=jev` 和 `TYPESAFE_API_KEY`。仅 Jev 需要该密钥且不需要宿主 Sampling。工具 `engine` 参数不能覆盖进程配置。不存在第二个辅助模型。旧 `maxTextCalls`、`maxRecoveryDecisions` 参数仅兼容已有调用且不生效;`BROWSER_OPERATE_TEXT_MODEL` 已不使用。输出 `textCalls:[]`、`recoveryDecisions:0`。
|
|
44
|
+
|
|
45
|
+
停滞或复杂障碍返回 `needs_reasoning`,调用方据实际观察决定下一步,不能盲目重跑。`model_done` 是未验证的完成提议:调用方必须对照完整原始目标统一验收 `finalObservation` 和页面结果。`resolvedValues` 是执行记录;同名字段、搜索框和候选文字不能冒充最终值。如果 `observationFresh:false`,或存在截断、覆盖缺口、冲突,补一次定向只读观察。将错误整理成局部修正任务,保留正确字段;修正后仍要核对原目标。所有验收和返工成本都计入端到端测试,不把快速失败当作提速。
|
|
46
|
+
|
|
47
|
+
`strategy:"fields"` 保留按完整资料顺序填写的旧路径,也没有文本助手。两种模式均保留 `verified:false`,不宣称平台已接受最终提交。
|
|
48
|
+
|
|
7
49
|
## 执行一个组合脚本
|
|
8
50
|
|
|
9
51
|
先通过 `list_pages` 获取准确 pageId。脚本固定在一个 browserInstance 的一个页面,执行期间复用连接并串行持有实例操作锁。不同实例可并行,页面诊断出口不被脚本占住。
|
|
@@ -26,6 +68,12 @@
|
|
|
26
68
|
|
|
27
69
|
默认上限:30 秒、100 次 helper 调用、32MiB 脚本堆,文本结果/日志/轨迹合计 64KiB。`timeoutMs`、`maxCalls` 只能调低。截图单独保存为本地产物,返回 `artifacts` 路径;最多 10 张。脚本环境没有文件读写、模块导入、`process`、`fetch`、裸 CDP 或页面 `evaluate`。
|
|
28
70
|
|
|
71
|
+
`page` 不是 Playwright。定位器构造器返回普通定位数据,没有 `.all()`、`.count()`、`.fill()`、`.textContent()` 或 `.getAttribute()` 方法;使用 `page.count(target)`、`page.fill(target,text)`、`page.read(target,{attribute})`。不支持 `page.url()` 或 `:visible` 等 Playwright 选择器扩展。遇到 `SCRIPT_ERROR` 时先检查已执行的 `actions`,不要重放已成功的操作。
|
|
72
|
+
|
|
73
|
+
陌生表单先调用独立的 `browser_snapshot`;空白 `scope` 与省略相同。脚本内 `page.snapshot()` 受 origin 限制不展开 iframe,可能只看到导航栏。此时使用独立快照返回的 `ref` 与 `snapshotId` 构造 `page.ref(ref,snapshotId)`,它会保留 frame 身份并继续执行现有 frame/origin 检查。不要猜测 iframe 的下标,也不要仅凭脚本内快照断言表单不存在。
|
|
74
|
+
|
|
75
|
+
字段可能以摘要、编辑或补充入口呈现,输入控件要展开后才出现。用户要求填写某字段时,先检查与它明确相关的已观察入口,再决定是否缺少可操作路径;必要时滚动对应区域。用户明确指定的字段即使在页面上标为可选,也需要完成或清楚报告真实阻塞原因。只提供部分表单值的快照不是“字段不支持”的证据。
|
|
76
|
+
|
|
29
77
|
## Helpers
|
|
30
78
|
|
|
31
79
|
| Helper | 用途 |
|
|
@@ -139,3 +187,7 @@ export async function readProfile(
|
|
|
139
187
|
```
|
|
140
188
|
|
|
141
189
|
`BrowserProgramDriver` 属于宿主信任边界。直接嵌入时,调用方负责实例锁、整段审批、当前站点/策略检查,以及 `close()` 中断连接;内核不会为任意自定义 driver 补齐这些能力。生产 MCP 入口使用 `BrowserScriptPageDriver` 与 `executeBrowserTool` 统一完成这些检查。执行结束会关闭 driver,不应跨执行复用已关闭对象。
|
|
190
|
+
|
|
191
|
+
决策请求会去重通用说明和资料表,并按字节预算缩短模型预览;保留原始 goal、所有动作 ID 与执行用完整文本。预览被截断不代表字段不存在,最终观察不使用这种压缩视图。同一决策请求另有一个整项完成判断,仅在动作建议 DONE 时使用;它可指出未完成要求或尚未处理的面板,不额外调用宿主模型,也不逐字段增加检查调用。
|
|
192
|
+
|
|
193
|
+
调用方必须原样转发用户目标和原样类文本,不能自行增删换行。原文只有一段时不得为了展示而重排成两段;最终核验包含这一点。
|
package/references/env.yaml
CHANGED
|
@@ -22,6 +22,10 @@ required:
|
|
|
22
22
|
purpose: 招聘事件 Open API Bearer token;公开 npm 包不内置兜底 token,必须由部署环境提供
|
|
23
23
|
example: token_xxx
|
|
24
24
|
optional:
|
|
25
|
+
- name: TYPESAFE_API_KEY
|
|
26
|
+
secret: true
|
|
27
|
+
purpose: TypeSafe 官方 Jev API 密钥,仅 browser.operate.engine 为 jev 时需要;可在 roll.config.yaml 的 agents.env.browser-use-agent.TYPESAFE_API_KEY 或配置台 Agent 环境变量中填写
|
|
28
|
+
example: token_xxx
|
|
25
29
|
- name: REPLY_AUTHORITY_TIMEOUT_MS
|
|
26
30
|
secret: false
|
|
27
31
|
type: number
|
|
@@ -80,6 +84,13 @@ optional:
|
|
|
80
84
|
sourcePath: [browser, instances]
|
|
81
85
|
purpose: Roll core 注入的多浏览器实例声明 JSON;由 `browser.instances` 派生,包含每个实例的 profile、CDP 端口、session 目录、可选 profileName/profileColor/windowBounds 和招聘事件归因 ID;orchestrator 不应手工拼接,手工配置时必须保持 cdpPort/userDataDir 唯一
|
|
82
86
|
example: '{"defaultInstance":"boss-a","instances":{"boss-a":{"platform":"zhipin","mode":"managed-cdp","cdpPort":9222,"userDataDir":"/tmp/roll-browser/profiles/boss-a","profileName":"boss-a","profileColor":"#2563EB","windowBounds":{"x":0,"y":0,"width":680,"height":1000},"trackingAgentId":"zhipin-boss-a"}}}'
|
|
87
|
+
- name: BROWSER_OPERATE_ENGINE
|
|
88
|
+
secret: false
|
|
89
|
+
configurable: false
|
|
90
|
+
sourcePath: [browser, operate, engine]
|
|
91
|
+
purpose: Roll core 从 browser.operate.engine 注入的 browser_operate 决策引擎;默认 sampling,jev 快速模式需要 TYPESAFE_API_KEY;不在 Agent 环境变量中手工填写
|
|
92
|
+
default: sampling
|
|
93
|
+
example: jev
|
|
83
94
|
- name: BROWSER_VISUAL_CURSOR
|
|
84
95
|
secret: false
|
|
85
96
|
type: boolean
|