@roll-agent/browser-use-agent 0.25.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +45 -16
- package/dist/browser-execution.d.ts +19 -0
- package/dist/browser-observation.d.ts +14 -0
- package/dist/browser-script-approval.d.ts +18 -0
- package/dist/index.js +1 -1
- package/dist/reply-authority/schemas.d.ts +2 -2
- package/dist/tools/browser-dom-action-candidates.d.ts +1 -0
- package/dist/tools/browser-execute.d.ts +152 -0
- package/dist/tools/browser-ref-frame-guard.d.ts +8 -0
- package/dist/tools/browser-ref-schemas.d.ts +6 -6
- package/dist/tools/browser-snapshot.d.ts +22 -3
- package/dist/tools/browser-workflows.d.ts +670 -0
- package/dist/tools/click-ref.d.ts +2 -1
- package/dist/tools/type-ref.d.ts +2 -1
- package/dist/tools/zhipin-diagnose-browser-state.d.ts +4 -4
- package/dist/tools/zhipin-generate-reply-preview.d.ts +2 -2
- package/dist/tools/zhipin-list-recommend-jobs.d.ts +2 -2
- package/dist/tools/zhipin-select-recommend-job.d.ts +6 -6
- package/dist/tools/zhipin-send-prepared-reply.d.ts +3 -3
- package/dist/workflows/parameters.d.ts +8 -0
- package/dist/workflows/store.d.ts +319 -0
- package/package.json +4 -4
- package/references/browser-exploration.md +141 -0
- package/references/generic-browser-refs.md +77 -25
|
@@ -15,7 +15,7 @@ declare const StorageEntrySummarySchema: z.ZodObject<{
|
|
|
15
15
|
key: string;
|
|
16
16
|
area: "localStorage" | "sessionStorage";
|
|
17
17
|
valueLength: number;
|
|
18
|
-
valueKind: "string" | "
|
|
18
|
+
valueKind: "string" | "json" | "empty";
|
|
19
19
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
20
20
|
jsonTopLevelKeys?: string[] | undefined;
|
|
21
21
|
jsonArrayLength?: number | undefined;
|
|
@@ -26,7 +26,7 @@ declare const StorageEntrySummarySchema: z.ZodObject<{
|
|
|
26
26
|
key: string;
|
|
27
27
|
area: "localStorage" | "sessionStorage";
|
|
28
28
|
valueLength: number;
|
|
29
|
-
valueKind: "string" | "
|
|
29
|
+
valueKind: "string" | "json" | "empty";
|
|
30
30
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
31
31
|
jsonTopLevelKeys?: string[] | undefined;
|
|
32
32
|
jsonArrayLength?: number | undefined;
|
|
@@ -304,7 +304,7 @@ export declare const zhipinDiagnoseBrowserState: import("@roll-agent/sdk").ToolD
|
|
|
304
304
|
key: string;
|
|
305
305
|
area: "localStorage" | "sessionStorage";
|
|
306
306
|
valueLength: number;
|
|
307
|
-
valueKind: "string" | "
|
|
307
|
+
valueKind: "string" | "json" | "empty";
|
|
308
308
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
309
309
|
jsonTopLevelKeys?: string[] | undefined;
|
|
310
310
|
jsonArrayLength?: number | undefined;
|
|
@@ -316,7 +316,7 @@ export declare const zhipinDiagnoseBrowserState: import("@roll-agent/sdk").ToolD
|
|
|
316
316
|
key: string;
|
|
317
317
|
area: "localStorage" | "sessionStorage";
|
|
318
318
|
valueLength: number;
|
|
319
|
-
valueKind: "string" | "
|
|
319
|
+
valueKind: "string" | "json" | "empty";
|
|
320
320
|
jsonKind?: "string" | "number" | "boolean" | "object" | "null" | "array" | undefined;
|
|
321
321
|
jsonTopLevelKeys?: string[] | undefined;
|
|
322
322
|
jsonArrayLength?: number | undefined;
|
|
@@ -22,20 +22,20 @@ export declare const zhipinGenerateReplyPreview: import("@roll-agent/sdk").ToolD
|
|
|
22
22
|
candidateName?: string | undefined;
|
|
23
23
|
reasoning?: {
|
|
24
24
|
enabled: boolean;
|
|
25
|
+
scope?: "reply" | "all" | undefined;
|
|
25
26
|
effort?: "high" | "low" | "medium" | undefined;
|
|
26
|
-
scope?: "all" | "reply" | undefined;
|
|
27
27
|
} | undefined;
|
|
28
28
|
maxMessages?: number | undefined;
|
|
29
29
|
}, {
|
|
30
30
|
success: boolean;
|
|
31
31
|
error?: string | undefined;
|
|
32
|
+
elapsedMs?: number | undefined;
|
|
32
33
|
confidence?: number | undefined;
|
|
33
34
|
requestId?: string | undefined;
|
|
34
35
|
expiresAt?: number | undefined;
|
|
35
36
|
suggestedReply?: string | undefined;
|
|
36
37
|
stage?: string | undefined;
|
|
37
38
|
preparedReplyId?: string | undefined;
|
|
38
|
-
elapsedMs?: number | undefined;
|
|
39
39
|
clientTimeoutMs?: number | undefined;
|
|
40
40
|
lastStartedPhase?: string | undefined;
|
|
41
41
|
activePhase?: string | undefined;
|
|
@@ -14,16 +14,16 @@ export declare const zhipinListRecommendJobs: import("@roll-agent/sdk").ToolDefi
|
|
|
14
14
|
canSwitch: boolean;
|
|
15
15
|
jobs: {
|
|
16
16
|
value: string;
|
|
17
|
-
index: number;
|
|
18
17
|
label: string;
|
|
18
|
+
index: number;
|
|
19
19
|
isCurrent: boolean;
|
|
20
20
|
jobRef?: string | undefined;
|
|
21
21
|
}[];
|
|
22
22
|
error?: string | undefined;
|
|
23
23
|
current?: {
|
|
24
24
|
value: string;
|
|
25
|
-
index: number;
|
|
26
25
|
label: string;
|
|
26
|
+
index: number;
|
|
27
27
|
isCurrent: boolean;
|
|
28
28
|
jobRef?: string | undefined;
|
|
29
29
|
} | undefined;
|
|
@@ -18,11 +18,11 @@ export declare const zhipinSelectRecommendJob: import("@roll-agent/sdk").ToolDef
|
|
|
18
18
|
}, {
|
|
19
19
|
options: {
|
|
20
20
|
value: string;
|
|
21
|
-
index: number;
|
|
22
21
|
label: string;
|
|
22
|
+
index: number;
|
|
23
23
|
isCurrent: boolean;
|
|
24
24
|
}[];
|
|
25
|
-
status: "not_found" | "
|
|
25
|
+
status: "not_found" | "selected" | "recommend_not_ready" | "already_selected" | "selector_not_found";
|
|
26
26
|
success: boolean;
|
|
27
27
|
requested: {
|
|
28
28
|
index?: number | undefined;
|
|
@@ -35,16 +35,16 @@ export declare const zhipinSelectRecommendJob: import("@roll-agent/sdk").ToolDef
|
|
|
35
35
|
};
|
|
36
36
|
matchedCount: number;
|
|
37
37
|
error?: string | undefined;
|
|
38
|
-
|
|
38
|
+
selected?: {
|
|
39
39
|
value: string;
|
|
40
|
-
index: number;
|
|
41
40
|
label: string;
|
|
41
|
+
index: number;
|
|
42
42
|
isCurrent: boolean;
|
|
43
43
|
} | undefined;
|
|
44
|
-
|
|
44
|
+
current?: {
|
|
45
45
|
value: string;
|
|
46
|
-
index: number;
|
|
47
46
|
label: string;
|
|
47
|
+
index: number;
|
|
48
48
|
isCurrent: boolean;
|
|
49
49
|
} | undefined;
|
|
50
50
|
}>;
|
|
@@ -12,15 +12,15 @@ export declare const zhipinSendPreparedReply: import("@roll-agent/sdk").ToolDefi
|
|
|
12
12
|
browserActionApproval?: {
|
|
13
13
|
id: string;
|
|
14
14
|
} | undefined;
|
|
15
|
+
toolActionApproval?: {
|
|
16
|
+
id: string;
|
|
17
|
+
} | undefined;
|
|
15
18
|
variantDecision?: {
|
|
16
19
|
reason: string;
|
|
17
20
|
chosenOption: "option_1" | "option_2";
|
|
18
21
|
confirmedFindingCodes?: ("too_many_questions" | "audit_tone" | "reply_overpacked" | "off_axis_fact_disclosure")[] | undefined;
|
|
19
22
|
judgeModel?: string | undefined;
|
|
20
23
|
} | undefined;
|
|
21
|
-
toolActionApproval?: {
|
|
22
|
-
id: string;
|
|
23
|
-
} | undefined;
|
|
24
24
|
skipVariantJudge?: boolean | undefined;
|
|
25
25
|
}, {
|
|
26
26
|
success: boolean;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Deliberately small JSON Schema subset: unsupported keywords fail closed. */
|
|
2
|
+
export type JsonValue = null | boolean | number | string | JsonValue[] | {
|
|
3
|
+
[key: string]: JsonValue;
|
|
4
|
+
};
|
|
5
|
+
export declare function assertJsonValue(value: unknown, depth?: number): asserts value is JsonValue;
|
|
6
|
+
export declare function canonicalJson(value: unknown): string;
|
|
7
|
+
export declare function validateParameterSchema(schema: unknown, depth?: number): asserts schema is Record<string, unknown>;
|
|
8
|
+
export declare function validateParameters(schema: unknown, value: unknown): JsonValue;
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const workflowDraftSchema: z.ZodEffects<z.ZodObject<{
|
|
3
|
+
id: z.ZodString;
|
|
4
|
+
name: z.ZodString;
|
|
5
|
+
description: z.ZodString;
|
|
6
|
+
appliesTo: z.ZodObject<{
|
|
7
|
+
origins: z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">;
|
|
8
|
+
pathPrefix: z.ZodOptional<z.ZodString>;
|
|
9
|
+
}, "strict", z.ZodTypeAny, {
|
|
10
|
+
origins: string[];
|
|
11
|
+
pathPrefix?: string | undefined;
|
|
12
|
+
}, {
|
|
13
|
+
origins: string[];
|
|
14
|
+
pathPrefix?: string | undefined;
|
|
15
|
+
}>;
|
|
16
|
+
source: z.ZodString;
|
|
17
|
+
parameterSchema: z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodUnknown>, Record<string, unknown>, Record<string, unknown>>;
|
|
18
|
+
capabilities: z.ZodArray<z.ZodEnum<["read", "interact", "navigate", "capture"]>, "many">;
|
|
19
|
+
allowedOrigins: z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">;
|
|
20
|
+
preconditions: z.ZodDefault<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">>;
|
|
21
|
+
postconditions: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
|
|
22
|
+
notes: z.ZodDefault<z.ZodString>;
|
|
23
|
+
locatorExplanations: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
24
|
+
}, "strict", z.ZodTypeAny, {
|
|
25
|
+
id: string;
|
|
26
|
+
source: string;
|
|
27
|
+
name: string;
|
|
28
|
+
description: string;
|
|
29
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
30
|
+
allowedOrigins: string[];
|
|
31
|
+
preconditions: Record<string, unknown>[];
|
|
32
|
+
postconditions: Record<string, unknown>[];
|
|
33
|
+
appliesTo: {
|
|
34
|
+
origins: string[];
|
|
35
|
+
pathPrefix?: string | undefined;
|
|
36
|
+
};
|
|
37
|
+
parameterSchema: Record<string, unknown>;
|
|
38
|
+
notes: string;
|
|
39
|
+
locatorExplanations: string[];
|
|
40
|
+
}, {
|
|
41
|
+
id: string;
|
|
42
|
+
source: string;
|
|
43
|
+
name: string;
|
|
44
|
+
description: string;
|
|
45
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
46
|
+
allowedOrigins: string[];
|
|
47
|
+
postconditions: Record<string, unknown>[];
|
|
48
|
+
appliesTo: {
|
|
49
|
+
origins: string[];
|
|
50
|
+
pathPrefix?: string | undefined;
|
|
51
|
+
};
|
|
52
|
+
parameterSchema: Record<string, unknown>;
|
|
53
|
+
preconditions?: Record<string, unknown>[] | undefined;
|
|
54
|
+
notes?: string | undefined;
|
|
55
|
+
locatorExplanations?: string[] | undefined;
|
|
56
|
+
}>, {
|
|
57
|
+
id: string;
|
|
58
|
+
source: string;
|
|
59
|
+
name: string;
|
|
60
|
+
description: string;
|
|
61
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
62
|
+
allowedOrigins: string[];
|
|
63
|
+
preconditions: Record<string, unknown>[];
|
|
64
|
+
postconditions: Record<string, unknown>[];
|
|
65
|
+
appliesTo: {
|
|
66
|
+
origins: string[];
|
|
67
|
+
pathPrefix?: string | undefined;
|
|
68
|
+
};
|
|
69
|
+
parameterSchema: Record<string, unknown>;
|
|
70
|
+
notes: string;
|
|
71
|
+
locatorExplanations: string[];
|
|
72
|
+
}, {
|
|
73
|
+
id: string;
|
|
74
|
+
source: string;
|
|
75
|
+
name: string;
|
|
76
|
+
description: string;
|
|
77
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
78
|
+
allowedOrigins: string[];
|
|
79
|
+
postconditions: Record<string, unknown>[];
|
|
80
|
+
appliesTo: {
|
|
81
|
+
origins: string[];
|
|
82
|
+
pathPrefix?: string | undefined;
|
|
83
|
+
};
|
|
84
|
+
parameterSchema: Record<string, unknown>;
|
|
85
|
+
preconditions?: Record<string, unknown>[] | undefined;
|
|
86
|
+
notes?: string | undefined;
|
|
87
|
+
locatorExplanations?: string[] | undefined;
|
|
88
|
+
}>;
|
|
89
|
+
export type WorkflowDraftInput = z.input<typeof workflowDraftSchema>;
|
|
90
|
+
export type WorkflowDraft = z.output<typeof workflowDraftSchema>;
|
|
91
|
+
export declare const workflowStatusSchema: z.ZodEnum<["draft", "active", "disabled", "suspended"]>;
|
|
92
|
+
declare const receiptSchema: z.ZodObject<{
|
|
93
|
+
compiled: z.ZodLiteral<true>;
|
|
94
|
+
success: z.ZodLiteral<true>;
|
|
95
|
+
verifiedAssertions: z.ZodNumber;
|
|
96
|
+
executionDigest: z.ZodString;
|
|
97
|
+
}, "strict", z.ZodTypeAny, {
|
|
98
|
+
success: true;
|
|
99
|
+
verifiedAssertions: number;
|
|
100
|
+
executionDigest: string;
|
|
101
|
+
compiled: true;
|
|
102
|
+
}, {
|
|
103
|
+
success: true;
|
|
104
|
+
verifiedAssertions: number;
|
|
105
|
+
executionDigest: string;
|
|
106
|
+
compiled: true;
|
|
107
|
+
}>;
|
|
108
|
+
export type WorkflowValidationReceipt = z.infer<typeof receiptSchema>;
|
|
109
|
+
declare const stateSchema: z.ZodObject<{
|
|
110
|
+
status: z.ZodEnum<["draft", "active", "disabled", "suspended"]>;
|
|
111
|
+
validations: z.ZodArray<z.ZodObject<{
|
|
112
|
+
compiled: z.ZodLiteral<true>;
|
|
113
|
+
success: z.ZodLiteral<true>;
|
|
114
|
+
verifiedAssertions: z.ZodNumber;
|
|
115
|
+
executionDigest: z.ZodString;
|
|
116
|
+
} & {
|
|
117
|
+
version: z.ZodString;
|
|
118
|
+
at: z.ZodString;
|
|
119
|
+
}, "strict", z.ZodTypeAny, {
|
|
120
|
+
at: string;
|
|
121
|
+
success: true;
|
|
122
|
+
verifiedAssertions: number;
|
|
123
|
+
version: string;
|
|
124
|
+
executionDigest: string;
|
|
125
|
+
compiled: true;
|
|
126
|
+
}, {
|
|
127
|
+
at: string;
|
|
128
|
+
success: true;
|
|
129
|
+
verifiedAssertions: number;
|
|
130
|
+
version: string;
|
|
131
|
+
executionDigest: string;
|
|
132
|
+
compiled: true;
|
|
133
|
+
}>, "many">;
|
|
134
|
+
updatedAt: z.ZodString;
|
|
135
|
+
}, "strict", z.ZodTypeAny, {
|
|
136
|
+
status: "disabled" | "draft" | "active" | "suspended";
|
|
137
|
+
validations: {
|
|
138
|
+
at: string;
|
|
139
|
+
success: true;
|
|
140
|
+
verifiedAssertions: number;
|
|
141
|
+
version: string;
|
|
142
|
+
executionDigest: string;
|
|
143
|
+
compiled: true;
|
|
144
|
+
}[];
|
|
145
|
+
updatedAt: string;
|
|
146
|
+
}, {
|
|
147
|
+
status: "disabled" | "draft" | "active" | "suspended";
|
|
148
|
+
validations: {
|
|
149
|
+
at: string;
|
|
150
|
+
success: true;
|
|
151
|
+
verifiedAssertions: number;
|
|
152
|
+
version: string;
|
|
153
|
+
executionDigest: string;
|
|
154
|
+
compiled: true;
|
|
155
|
+
}[];
|
|
156
|
+
updatedAt: string;
|
|
157
|
+
}>;
|
|
158
|
+
export type WorkflowState = z.infer<typeof stateSchema>;
|
|
159
|
+
declare const versionSchema: z.ZodObject<{
|
|
160
|
+
version: z.ZodString;
|
|
161
|
+
createdAt: z.ZodString;
|
|
162
|
+
draft: z.ZodEffects<z.ZodObject<{
|
|
163
|
+
id: z.ZodString;
|
|
164
|
+
name: z.ZodString;
|
|
165
|
+
description: z.ZodString;
|
|
166
|
+
appliesTo: z.ZodObject<{
|
|
167
|
+
origins: z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">;
|
|
168
|
+
pathPrefix: z.ZodOptional<z.ZodString>;
|
|
169
|
+
}, "strict", z.ZodTypeAny, {
|
|
170
|
+
origins: string[];
|
|
171
|
+
pathPrefix?: string | undefined;
|
|
172
|
+
}, {
|
|
173
|
+
origins: string[];
|
|
174
|
+
pathPrefix?: string | undefined;
|
|
175
|
+
}>;
|
|
176
|
+
source: z.ZodString;
|
|
177
|
+
parameterSchema: z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodUnknown>, Record<string, unknown>, Record<string, unknown>>;
|
|
178
|
+
capabilities: z.ZodArray<z.ZodEnum<["read", "interact", "navigate", "capture"]>, "many">;
|
|
179
|
+
allowedOrigins: z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">;
|
|
180
|
+
preconditions: z.ZodDefault<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">>;
|
|
181
|
+
postconditions: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
|
|
182
|
+
notes: z.ZodDefault<z.ZodString>;
|
|
183
|
+
locatorExplanations: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
184
|
+
}, "strict", z.ZodTypeAny, {
|
|
185
|
+
id: string;
|
|
186
|
+
source: string;
|
|
187
|
+
name: string;
|
|
188
|
+
description: string;
|
|
189
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
190
|
+
allowedOrigins: string[];
|
|
191
|
+
preconditions: Record<string, unknown>[];
|
|
192
|
+
postconditions: Record<string, unknown>[];
|
|
193
|
+
appliesTo: {
|
|
194
|
+
origins: string[];
|
|
195
|
+
pathPrefix?: string | undefined;
|
|
196
|
+
};
|
|
197
|
+
parameterSchema: Record<string, unknown>;
|
|
198
|
+
notes: string;
|
|
199
|
+
locatorExplanations: string[];
|
|
200
|
+
}, {
|
|
201
|
+
id: string;
|
|
202
|
+
source: string;
|
|
203
|
+
name: string;
|
|
204
|
+
description: string;
|
|
205
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
206
|
+
allowedOrigins: string[];
|
|
207
|
+
postconditions: Record<string, unknown>[];
|
|
208
|
+
appliesTo: {
|
|
209
|
+
origins: string[];
|
|
210
|
+
pathPrefix?: string | undefined;
|
|
211
|
+
};
|
|
212
|
+
parameterSchema: Record<string, unknown>;
|
|
213
|
+
preconditions?: Record<string, unknown>[] | undefined;
|
|
214
|
+
notes?: string | undefined;
|
|
215
|
+
locatorExplanations?: string[] | undefined;
|
|
216
|
+
}>, {
|
|
217
|
+
id: string;
|
|
218
|
+
source: string;
|
|
219
|
+
name: string;
|
|
220
|
+
description: string;
|
|
221
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
222
|
+
allowedOrigins: string[];
|
|
223
|
+
preconditions: Record<string, unknown>[];
|
|
224
|
+
postconditions: Record<string, unknown>[];
|
|
225
|
+
appliesTo: {
|
|
226
|
+
origins: string[];
|
|
227
|
+
pathPrefix?: string | undefined;
|
|
228
|
+
};
|
|
229
|
+
parameterSchema: Record<string, unknown>;
|
|
230
|
+
notes: string;
|
|
231
|
+
locatorExplanations: string[];
|
|
232
|
+
}, {
|
|
233
|
+
id: string;
|
|
234
|
+
source: string;
|
|
235
|
+
name: string;
|
|
236
|
+
description: string;
|
|
237
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
238
|
+
allowedOrigins: string[];
|
|
239
|
+
postconditions: Record<string, unknown>[];
|
|
240
|
+
appliesTo: {
|
|
241
|
+
origins: string[];
|
|
242
|
+
pathPrefix?: string | undefined;
|
|
243
|
+
};
|
|
244
|
+
parameterSchema: Record<string, unknown>;
|
|
245
|
+
preconditions?: Record<string, unknown>[] | undefined;
|
|
246
|
+
notes?: string | undefined;
|
|
247
|
+
locatorExplanations?: string[] | undefined;
|
|
248
|
+
}>;
|
|
249
|
+
}, "strict", z.ZodTypeAny, {
|
|
250
|
+
version: string;
|
|
251
|
+
createdAt: string;
|
|
252
|
+
draft: {
|
|
253
|
+
id: string;
|
|
254
|
+
source: string;
|
|
255
|
+
name: string;
|
|
256
|
+
description: string;
|
|
257
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
258
|
+
allowedOrigins: string[];
|
|
259
|
+
preconditions: Record<string, unknown>[];
|
|
260
|
+
postconditions: Record<string, unknown>[];
|
|
261
|
+
appliesTo: {
|
|
262
|
+
origins: string[];
|
|
263
|
+
pathPrefix?: string | undefined;
|
|
264
|
+
};
|
|
265
|
+
parameterSchema: Record<string, unknown>;
|
|
266
|
+
notes: string;
|
|
267
|
+
locatorExplanations: string[];
|
|
268
|
+
};
|
|
269
|
+
}, {
|
|
270
|
+
version: string;
|
|
271
|
+
createdAt: string;
|
|
272
|
+
draft: {
|
|
273
|
+
id: string;
|
|
274
|
+
source: string;
|
|
275
|
+
name: string;
|
|
276
|
+
description: string;
|
|
277
|
+
capabilities: ("navigate" | "read" | "interact" | "capture")[];
|
|
278
|
+
allowedOrigins: string[];
|
|
279
|
+
postconditions: Record<string, unknown>[];
|
|
280
|
+
appliesTo: {
|
|
281
|
+
origins: string[];
|
|
282
|
+
pathPrefix?: string | undefined;
|
|
283
|
+
};
|
|
284
|
+
parameterSchema: Record<string, unknown>;
|
|
285
|
+
preconditions?: Record<string, unknown>[] | undefined;
|
|
286
|
+
notes?: string | undefined;
|
|
287
|
+
locatorExplanations?: string[] | undefined;
|
|
288
|
+
};
|
|
289
|
+
}>;
|
|
290
|
+
export type WorkflowVersion = z.infer<typeof versionSchema>;
|
|
291
|
+
export type StoredWorkflow = WorkflowVersion & {
|
|
292
|
+
state: WorkflowState;
|
|
293
|
+
};
|
|
294
|
+
/** Host-owned store. Never expose recordValidation directly as an MCP operation. */
|
|
295
|
+
export declare class WorkflowStore {
|
|
296
|
+
readonly root: string;
|
|
297
|
+
private readonly managedBase;
|
|
298
|
+
constructor(options?: {
|
|
299
|
+
root?: string;
|
|
300
|
+
});
|
|
301
|
+
private directory;
|
|
302
|
+
private read;
|
|
303
|
+
private write;
|
|
304
|
+
private locked;
|
|
305
|
+
private load;
|
|
306
|
+
saveDraft(input: WorkflowDraftInput): Promise<StoredWorkflow>;
|
|
307
|
+
getVersion(id: string, version: string): Promise<StoredWorkflow>;
|
|
308
|
+
recordValidation(id: string, version: string, receipt: WorkflowValidationReceipt): Promise<WorkflowState>;
|
|
309
|
+
setStatus(id: string, version: string, status: "active" | "disabled" | "suspended"): Promise<WorkflowState>;
|
|
310
|
+
listApplicableActive(url: string): Promise<Array<{
|
|
311
|
+
id: string;
|
|
312
|
+
version: string;
|
|
313
|
+
name: string;
|
|
314
|
+
description: string;
|
|
315
|
+
appliesTo: WorkflowDraft["appliesTo"];
|
|
316
|
+
parameterSchema: WorkflowDraft["parameterSchema"];
|
|
317
|
+
}>>;
|
|
318
|
+
}
|
|
319
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@roll-agent/browser-use-agent",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.26.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -44,10 +44,10 @@
|
|
|
44
44
|
}
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"zod": "^3.25.76",
|
|
48
47
|
"@roll-agent/reply-authority-client": "0.7.0",
|
|
49
|
-
"@roll-agent/sdk": "0.5.
|
|
50
|
-
"@roll-agent/browser": "0.
|
|
48
|
+
"@roll-agent/sdk": "0.5.1",
|
|
49
|
+
"@roll-agent/browser": "0.11.0",
|
|
50
|
+
"zod": "^3.25.76"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@types/node": "^22.0.0"
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# 通用页面探索
|
|
2
|
+
|
|
3
|
+
已建模的 BOSS 操作优先使用 `zhipin_*`。本文描述未知页面的补充能力,不替代签名回复、平台语义校验或现有预编排。
|
|
4
|
+
|
|
5
|
+
单步 `browser_snapshot` / `click_ref` / `type_ref` 的区域观察、上下文选择和严格 ref 用法见 [通用 ref 操作](generic-browser-refs.md)。
|
|
6
|
+
|
|
7
|
+
## 执行一个组合脚本
|
|
8
|
+
|
|
9
|
+
先通过 `list_pages` 获取准确 pageId。脚本固定在一个 browserInstance 的一个页面,执行期间复用连接并串行持有实例操作锁。不同实例可并行,页面诊断出口不被脚本占住。
|
|
10
|
+
|
|
11
|
+
调用 `browser_execute` 的示例输入:
|
|
12
|
+
|
|
13
|
+
以下是针对假设存在 `#profile` 表单的接口示例;example.com 本身不提供该表单。实际调用需替换为目标页面、已确认的定位器及其 origin。
|
|
14
|
+
|
|
15
|
+
```json
|
|
16
|
+
{
|
|
17
|
+
"pageId": "从 list_pages 取得的页面 ID",
|
|
18
|
+
"capabilities": ["read", "interact", "navigate"],
|
|
19
|
+
"allowedOrigins": ["https://example.com"],
|
|
20
|
+
"args": { "name": "Ada" },
|
|
21
|
+
"source": "const field = page.locator('#name', {scope:'form#profile'}); await page.fill(field, args.name, {expect:{target:field,value:args.name}}); await page.click(page.getByRole('button',{name:'Save',scope:'form#profile'})); await page.expect({target:page.locator('#result'),text:'Saved',match:'contains'}); return {saved:true};"
|
|
22
|
+
}
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
通过 CLI 执行时,将该 JSON 保存到临时输入文件,再使用 `roll run browser-use-agent browser_execute --input-file <输入文件> --json`。也可直接使用 `--input-json`。参数属于本次执行,不会自动保存到站点经验中。
|
|
26
|
+
|
|
27
|
+
默认上限:30 秒、100 次 helper 调用、32MiB 脚本堆,文本结果/日志/轨迹合计 64KiB。`timeoutMs`、`maxCalls` 只能调低。截图单独保存为本地产物,返回 `artifacts` 路径;最多 10 张。脚本环境没有文件读写、模块导入、`process`、`fetch`、裸 CDP 或页面 `evaluate`。
|
|
28
|
+
|
|
29
|
+
## Helpers
|
|
30
|
+
|
|
31
|
+
| Helper | 用途 |
|
|
32
|
+
| ---------------------------------------------------------------------- | ------------------------------------------------------------------ |
|
|
33
|
+
| `page.locator(css, {scope?,frameId?})` | 构造 CSS 定位,不执行查询 |
|
|
34
|
+
| `page.getByRole(role, {name,scope?,frameId?})` | 构造精确 role/name 定位 |
|
|
35
|
+
| `page.ref(ref, snapshotId)` | 使用当前页面最新 Snapshot 的严格 ref |
|
|
36
|
+
| `page.observe()` | 有界页面状态,适合决策点观察 |
|
|
37
|
+
| `page.snapshot({scope?})` | 按需语义快照,带上下文、locator 提示和覆盖缺口;受节点上限约束 |
|
|
38
|
+
| `page.read(target, {attribute?})` | 读取有界文本/值/可见状态;属性仅允许常见标签属性,密码值不返回 |
|
|
39
|
+
| `page.exists(target)` / `page.count(target)` | 条件分支和匹配数量;读取/操作目标要求唯一匹配 |
|
|
40
|
+
| `page.click(target, {expect?})` / `page.fill(target, text, {expect?})` | 点击或清空后输入,并可验证后置条件 |
|
|
41
|
+
| `page.hover(target, {expect?})` | 悬停并可验证状态变化 |
|
|
42
|
+
| `page.press(key, {target?,expect?})` | 常用页面按键;要求明确目标或本脚本建立的焦点,不开放浏览器快捷键 |
|
|
43
|
+
| `page.scroll(target, {dx?,dy?,expect?})` | 指定容器滚动,每轴最多 2000px |
|
|
44
|
+
| `page.goto(url, {expect?})` | 当前页面导航,需 navigate 能力 |
|
|
45
|
+
| `page.waitFor(condition, {timeoutMs?})` / `page.expect(...)` | 有期限地等待/断言,失败终止后续副作用 |
|
|
46
|
+
| `page.inspectControl(field, {panel?})` | 只读识别单选字段、关联面板、iframe 和当前渲染选项 |
|
|
47
|
+
| `page.choose(field, {label?,value?,panel?,timeoutMs?,expect?})` | 按精确标签或值选择一次,并验证字段结果;label/value 二选一 |
|
|
48
|
+
| `page.screenshot()` | 截图产物,需 capture 能力 |
|
|
49
|
+
|
|
50
|
+
所有脚本需要 `read` 能力,以便检查页面和目标。`interact` 包含点击、输入、按键、悬停、滚动和 `choose`;`inspectControl` 只需 `read`。已知链接或表单导航还需要 `navigate`。允许站点必须是明确的 HTTP(S) origin,并受到全局 domainAllowlist 约束。单页脚本不会自动切换到新标签页。
|
|
51
|
+
|
|
52
|
+
条件支持:`{target,state:'visible'}`(也支持 attached/absent/hidden/enabled/disabled/checked/unchecked)、`{target,text,match:'equals'|'contains'}`、`{target,value}`、`{url,match:'equals'|'startsWith'}`。执行级 `preconditions` / `postconditions` 使用相同格式。
|
|
53
|
+
|
|
54
|
+
## 通用单选控件
|
|
55
|
+
|
|
56
|
+
`page.inspectControl(field, {panel?})` 是只读观察;`page.choose(field, {label?,value?,panel?,timeoutMs?,expect?})` 是受策略和审批约束的交互。`label` 与 `value` 必须且只能提供一个,使用精确匹配。`timeoutMs` 默认 3000,上限 10000 毫秒,仍受整段脚本截止时间约束。
|
|
57
|
+
|
|
58
|
+
```js
|
|
59
|
+
// field 来自刚才观察的严格 ref;它保留页面及 iframe 归属。
|
|
60
|
+
const field = page.ref(args.ref, args.snapshotId);
|
|
61
|
+
const state = await page.inspectControl(field);
|
|
62
|
+
if (state.association === "ambiguous") return {needsExploration: true, state};
|
|
63
|
+
await page.choose(field, {label: args.label});
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
观察返回 `kind`、`association`、`triggerCss`、`panelCss`、`frameId`、`pageId`、当前 `value` / `text`、`expanded`、`multiple`、最多 100 个 `options` 和 `coverageWarnings`。每个选项包含 `label`、`value`、`selected`、`disabled` 与诊断用 CSS 路径。CSS 路径属于当前 DOM,不是持久化经验中可直接复用的稳定定位器。
|
|
67
|
+
|
|
68
|
+
关联顺序为原生 SELECT、显式 `panel`、`aria-controls` / `aria-owns`、同一控件附近唯一的结构选项组。不会越过相邻字段去认领另一菜单。没有关联属性的门户浮层可以在观察其身份后显式限定:
|
|
69
|
+
|
|
70
|
+
```js
|
|
71
|
+
await page.choose(page.locator('#field', {frameId: args.frameId}), {
|
|
72
|
+
label: args.label,
|
|
73
|
+
panel: '#visible-menu',
|
|
74
|
+
});
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
上述 CSS 是接口示例,实际值必须来自页面观察。重复 panel ID、多个候选组或同一面板中重复标签都停止执行。没有足够关联证据时返回 `control_unassociated`;工具可能已经打开控件,失败后需检查现场,不重放整段。
|
|
78
|
+
|
|
79
|
+
选择只执行一次。原生 SELECT 按具体选项选择并核对 selected 状态,避免不同标签共用 value 时选错;自定义控件要求可信的 selected 状态、对应字段值/显示文案变化,或调用者显式声明的 `expect`。面板消失和点击命令成功本身都不足以证明选中了目标。
|
|
80
|
+
|
|
81
|
+
级联菜单用明确的 `panel` 和每级 `expect` 分步编排:父级操作可声明子面板出现,最后一级验证最终字段值。没有足够结构证据时不自动展开任意深度的级联。原生或 ARIA 多选、超过选项上限、未渲染的虚拟列表项等返回限制或要求进一步探索;不会默认滚动全部列表。自定义列表的 `rendered_options_only` 提醒调用方当前结果不保证覆盖所有逻辑选项。
|
|
82
|
+
|
|
83
|
+
在 iframe 内,字段 locator 必须携带正确 `frameId`,或使用保留 frame 身份的严格 ref。输入前会检查同源 iframe 的祖先命中与焦点;外层遮挡返回 `target_obscured`,焦点改变返回 `focus_changed`,无法遍历或不支持的 CSS 变换返回 `coverage_gap`;跨源祖先同样停止,未声明允许的 origin 会先触发域名边界错误。这延续了“无法可靠验证就停止”的规则。
|
|
84
|
+
|
|
85
|
+
这些机制按原生标签、ARIA 关系、DOM 结构与可操作证据工作,不包含 BOSS 字段名、域名或组件类名。表单失配时重新观察当前浮层和字段,不用 reload 作为默认恢复操作。
|
|
86
|
+
|
|
87
|
+
## 理解结果和错误
|
|
88
|
+
|
|
89
|
+
- `status`:completed、failed、cancelled、timed_out;不是网站业务状态。
|
|
90
|
+
- `verification`:passed、not_requested、failed。未提供断言时不能把 completed 当作业务成功;最后一次副作用之后需要验证才能标记结果已验证。
|
|
91
|
+
- `actions` 保留各步骤是否执行、验证状态和耗时;`checks` 带验证结果及失败时的有界实际观察。
|
|
92
|
+
- `observation` 返回有限的页面变化;`value` 是脚本返回值;`logs` 是受限日志。不要将敏感内容写入日志。
|
|
93
|
+
- 取消或错误不回滚已经发生的操作;审批拒绝、目标失效、验证失败不能被 JS `try/catch` 转成成功并继续操作。
|
|
94
|
+
- `actionPolicy=confirm` 的批准在副作用前一次性完成,并绑定脚本、参数、实例、页面 document、站点、能力和限制。批准仅能使用一次。运行中策略变更会终止后续动作,不提供整段自动重放。
|
|
95
|
+
- Runtime 客户端会通过现有确认通道处理明确标记 `executionState:not_executed` 的审批请求;批准后仅补充返回的凭据续接一次,保持原始业务参数与实例资源锁。拒绝、取消、部分完成或第二次审批请求均不会自动重试。直接 CLI 调用仍返回结构化 `approvalRequest.retryInput`,由操作者确认后原样补入。
|
|
96
|
+
|
|
97
|
+
新入口中的 ref 绑定最新 snapshotId、browserInstance、pageId 与 documentId;刷新快照、导航、跨实例使用或节点脱离时都可能失效。已有 `click_ref` / `type_ref` 可附带 `snapshotId` 使用相同严格规则;旧调用仍兼容。
|
|
98
|
+
|
|
99
|
+
Snapshot 在 `coverageWarnings` 明确指出 canvas 或不能可靠读取的 frame 等缺口。脚本限定 origin 时当前实现保守跳过 frame 展开;必要时使用已支持的同源 frame 定位或截图观察,不把空语义树当作空页面。
|
|
100
|
+
|
|
101
|
+
## 经验闭环
|
|
102
|
+
|
|
103
|
+
工具:`browser_workflow_list({url})`、`browser_workflow_save_draft({draft})`、`browser_workflow_validate({id,version,pageId,args})`、`browser_workflow_set_status({id,version,status})`、`browser_workflow_run({id,version,pageId,args})`。
|
|
104
|
+
|
|
105
|
+
草稿包含 id、name、description、`appliesTo:{origins,pathPrefix?}`、source、parameterSchema、capabilities、allowedOrigins、preconditions、postconditions、notes 和 locatorExplanations。参数 schema 顶层为 object,仅支持有界的基本 JSON Schema 子集;不支持的关键字会被拒绝。业务输入通过 args 传入,草稿不要硬编码账号数据、秘密或本次 Snapshot ref。
|
|
106
|
+
|
|
107
|
+
每份内容产生不可变 version。保存只检查语法,不执行代码。验证显式运行该版本,成功且有结果断言才记录服务端验证凭据;调用者不能提交伪造成功记录。激活需要针对该 version 的单次显式批准,修改内容后重新保存/验证/批准。停用立即停止后续 helper;定位或断言失效会暂停推荐,需要重新探索。
|
|
108
|
+
|
|
109
|
+
默认库在 `~/.roll-agent/browser/workflows`,当前用户跨项目复用。记录只保存模板和验证摘要,不自动持久化运行参数、页面正文、消息内容或截图。发现只返回已启用且适用于 URL 的简短签名;不动态注册站点专属 MCP 工具。编排器负责从成功探索整理草稿,执行 Agent 不自行调用模型生成经验。
|
|
110
|
+
|
|
111
|
+
## 浏览器 SDK 嵌入
|
|
112
|
+
|
|
113
|
+
应用通常通过 MCP 调用 `browser_execute`,以复用实例锁、策略与审批。实现自有受信任宿主时,可从 `@roll-agent/browser` 导入执行内核:
|
|
114
|
+
|
|
115
|
+
```ts
|
|
116
|
+
import {
|
|
117
|
+
BrowserExecuteInputSchema,
|
|
118
|
+
executeBrowserProgram,
|
|
119
|
+
type BrowserProgramDriver,
|
|
120
|
+
} from "@roll-agent/browser";
|
|
121
|
+
|
|
122
|
+
export async function readProfile(
|
|
123
|
+
pageId: string,
|
|
124
|
+
driver: BrowserProgramDriver,
|
|
125
|
+
signal: AbortSignal,
|
|
126
|
+
) {
|
|
127
|
+
const input = BrowserExecuteInputSchema.parse({
|
|
128
|
+
pageId,
|
|
129
|
+
capabilities: ["read"],
|
|
130
|
+
allowedOrigins: ["https://example.com"],
|
|
131
|
+
source: `
|
|
132
|
+
const target = page.locator('#profile-name');
|
|
133
|
+
await page.expect({target, state: 'visible'});
|
|
134
|
+
return await page.read(target);
|
|
135
|
+
`,
|
|
136
|
+
});
|
|
137
|
+
return executeBrowserProgram(input, { driver, signal });
|
|
138
|
+
}
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
`BrowserProgramDriver` 属于宿主信任边界。直接嵌入时,调用方负责实例锁、整段审批、当前站点/策略检查,以及 `close()` 中断连接;内核不会为任意自定义 driver 补齐这些能力。生产 MCP 入口使用 `BrowserScriptPageDriver` 与 `executeBrowserTool` 统一完成这些检查。执行结束会关闭 driver,不应跨执行复用已关闭对象。
|