@coolkiller007/my-page-agent 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +89 -0
- package/UPSTREAM.md +97 -0
- package/dist/index.d.ts +873 -0
- package/dist/index.js +4436 -0
- package/dist/index.js.map +1 -0
- package/package.json +84 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,873 @@
|
|
|
1
|
+
import * as z from 'zod/v4';
|
|
2
|
+
|
|
3
|
+
declare interface ActionResult {
|
|
4
|
+
success: boolean;
|
|
5
|
+
message: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Agent activity - transient state for immediate UI feedback.
|
|
10
|
+
*
|
|
11
|
+
* Unlike historical events (which are persisted), activities are ephemeral
|
|
12
|
+
* and represent "what the agent is doing right now". UI components should
|
|
13
|
+
* listen to 'activity' events to show real-time feedback.
|
|
14
|
+
*
|
|
15
|
+
* Note: There is no 'idle' activity - absence of activity events means idle.
|
|
16
|
+
*/
|
|
17
|
+
export declare type AgentActivity = {
|
|
18
|
+
type: 'thinking';
|
|
19
|
+
} | {
|
|
20
|
+
type: 'executing';
|
|
21
|
+
tool: string;
|
|
22
|
+
input: unknown;
|
|
23
|
+
} | {
|
|
24
|
+
type: 'executed';
|
|
25
|
+
tool: string;
|
|
26
|
+
input: unknown;
|
|
27
|
+
output: string;
|
|
28
|
+
duration: number;
|
|
29
|
+
} | {
|
|
30
|
+
type: 'retrying';
|
|
31
|
+
attempt: number;
|
|
32
|
+
maxAttempts: number;
|
|
33
|
+
} | {
|
|
34
|
+
type: 'error';
|
|
35
|
+
message: string;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
declare interface AgentConfig extends LLMConfig {
|
|
39
|
+
language?: SupportedLanguage;
|
|
40
|
+
/**
|
|
41
|
+
* Maximum number of steps the agent can take per task.
|
|
42
|
+
* @default 40
|
|
43
|
+
*/
|
|
44
|
+
maxSteps?: number;
|
|
45
|
+
/**
|
|
46
|
+
* Custom tools to extend PageAgent capabilities
|
|
47
|
+
* @experimental
|
|
48
|
+
* @note You can also override or remove internal tools by using the same name.
|
|
49
|
+
* @see AgentTool
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* // override internal tool
|
|
53
|
+
* import { z } from 'zod/v4'
|
|
54
|
+
* import { tool } from '@coolkiller007/my-page-agent'
|
|
55
|
+
* const customTools = {
|
|
56
|
+
* ask_user: tool({
|
|
57
|
+
* description:
|
|
58
|
+
* 'Ask the user or parent model a question and wait for their answer. Use this if you need more information or clarification.',
|
|
59
|
+
* inputSchema: z.object({
|
|
60
|
+
* question: z.string(),
|
|
61
|
+
* }),
|
|
62
|
+
* execute: async function (this: PageAgent, input) {
|
|
63
|
+
* const answer = await do_some_thing(input.question)
|
|
64
|
+
* return "✅ Received user answer: " + answer
|
|
65
|
+
* },
|
|
66
|
+
* })
|
|
67
|
+
* }
|
|
68
|
+
*
|
|
69
|
+
* @example
|
|
70
|
+
* // remove internal tool
|
|
71
|
+
* const customTools = {
|
|
72
|
+
* ask_user: null // never ask user questions
|
|
73
|
+
* }
|
|
74
|
+
*/
|
|
75
|
+
customTools?: Record<string, AgentTool | null>;
|
|
76
|
+
/**
|
|
77
|
+
* Instructions to guide the agent's behavior
|
|
78
|
+
*/
|
|
79
|
+
instructions?: {
|
|
80
|
+
/**
|
|
81
|
+
* Global system-level instructions, applied to all tasks
|
|
82
|
+
*/
|
|
83
|
+
system?: string;
|
|
84
|
+
/**
|
|
85
|
+
* Dynamic page-level instructions callback
|
|
86
|
+
* Called before each step to get instructions for the current page
|
|
87
|
+
* @param url - Current page URL (window.location.href)
|
|
88
|
+
* @returns Instructions string, or undefined/null to skip
|
|
89
|
+
*/
|
|
90
|
+
getPageInstructions?: (url: string) => string | undefined | null;
|
|
91
|
+
};
|
|
92
|
+
/**
|
|
93
|
+
* Lifecycle hooks for task execution.
|
|
94
|
+
* @experimental API may change in future versions.
|
|
95
|
+
*
|
|
96
|
+
* All hooks receive the agent instance as first parameter.
|
|
97
|
+
*/
|
|
98
|
+
/**
|
|
99
|
+
* Called before each step execution.
|
|
100
|
+
* @experimental
|
|
101
|
+
* @param agent - The AgentRuntime instance
|
|
102
|
+
* @param stepCount - Current step number (0-indexed)
|
|
103
|
+
*/
|
|
104
|
+
onBeforeStep?: (agent: AgentRuntime, stepCount: number) => Promise<void> | void;
|
|
105
|
+
/**
|
|
106
|
+
* Called after each step execution.
|
|
107
|
+
* @experimental
|
|
108
|
+
* @param agent - The AgentRuntime instance
|
|
109
|
+
* @param history - Current history of events
|
|
110
|
+
*/
|
|
111
|
+
onAfterStep?: (agent: AgentRuntime, history: HistoricalEvent[]) => Promise<void> | void;
|
|
112
|
+
/**
|
|
113
|
+
* Called before task execution starts.
|
|
114
|
+
* @experimental
|
|
115
|
+
* @param agent - The AgentRuntime instance
|
|
116
|
+
*/
|
|
117
|
+
onBeforeTask?: (agent: AgentRuntime) => Promise<void> | void;
|
|
118
|
+
/**
|
|
119
|
+
* Called after task execution completes (success or failure).
|
|
120
|
+
* @experimental
|
|
121
|
+
* @param agent - The AgentRuntime instance
|
|
122
|
+
* @param result - The execution result
|
|
123
|
+
*/
|
|
124
|
+
onAfterTask?: (agent: AgentRuntime, result: ExecutionResult) => Promise<void> | void;
|
|
125
|
+
/**
|
|
126
|
+
* Called when the agent is disposed.
|
|
127
|
+
* @experimental
|
|
128
|
+
* @note This hook can block the disposal process if it's async.
|
|
129
|
+
* @param agent - The AgentRuntime instance
|
|
130
|
+
* @param reason - Optional reason for disposal
|
|
131
|
+
*/
|
|
132
|
+
onDispose?: (agent: AgentRuntime, reason?: string) => void;
|
|
133
|
+
/**
|
|
134
|
+
* @experimental
|
|
135
|
+
* Enable the experimental script execution tool that allows executing generated JavaScript code on the page.
|
|
136
|
+
* @note Can cause unpredictable side effects.
|
|
137
|
+
* @note May bypass some safe guards and data-masking mechanisms.
|
|
138
|
+
*/
|
|
139
|
+
experimentalScriptExecutionTool?: boolean;
|
|
140
|
+
/**
|
|
141
|
+
* @experimental
|
|
142
|
+
* Fetch /llms.txt from current site origin and include as context.
|
|
143
|
+
* Only fetched once per origin per task.
|
|
144
|
+
* @default false
|
|
145
|
+
*/
|
|
146
|
+
experimentalLlmsTxt?: boolean;
|
|
147
|
+
/**
|
|
148
|
+
* Transform page content before sending to LLM.
|
|
149
|
+
* Called after DOM extraction and simplification, before LLM invocation.
|
|
150
|
+
* Use cases: inspect extraction results, modify page info, mask sensitive data.
|
|
151
|
+
*
|
|
152
|
+
* @param content - Simplified page content that will be sent to LLM
|
|
153
|
+
* @returns Transformed content
|
|
154
|
+
*
|
|
155
|
+
* @example
|
|
156
|
+
* // Mask phone numbers
|
|
157
|
+
* transformPageContent: async (content) => {
|
|
158
|
+
* return content.replace(/1[3-9]\d{9}/g, '***********')
|
|
159
|
+
* }
|
|
160
|
+
*/
|
|
161
|
+
transformPageContent?: (content: string) => Promise<string> | string;
|
|
162
|
+
/**
|
|
163
|
+
* Completely override the default system prompt.
|
|
164
|
+
* @experimental Use with caution - incorrect prompts may break agent behavior.
|
|
165
|
+
*/
|
|
166
|
+
customSystemPrompt?: string;
|
|
167
|
+
/**
|
|
168
|
+
* Delay between steps in seconds.
|
|
169
|
+
* @default 0.4
|
|
170
|
+
*/
|
|
171
|
+
stepDelay?: number;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Error event - fatal error from LLM or execution
|
|
176
|
+
*/
|
|
177
|
+
declare interface AgentErrorEvent {
|
|
178
|
+
type: 'error';
|
|
179
|
+
message: string;
|
|
180
|
+
rawResponse?: unknown;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Agent reflection state - the reflection-before-action model
|
|
185
|
+
*
|
|
186
|
+
* Every tool call must first reflect on:
|
|
187
|
+
* - evaluation_previous_goal: How well did the previous action achieve its goal?
|
|
188
|
+
* - memory: Key information to remember for future steps
|
|
189
|
+
* - next_goal: What should be accomplished in the next action?
|
|
190
|
+
*/
|
|
191
|
+
declare interface AgentReflection {
|
|
192
|
+
evaluation_previous_goal: string;
|
|
193
|
+
memory: string;
|
|
194
|
+
next_goal: string;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* AI agent for browser automation.
|
|
199
|
+
*
|
|
200
|
+
* @remarks
|
|
201
|
+
* ## Re-act Agent Loop
|
|
202
|
+
* - step
|
|
203
|
+
* - observe (gather information about current environment and context)
|
|
204
|
+
* - think (LLM calling)
|
|
205
|
+
* - reflection (evaluate history, generate memory, short-term planning)
|
|
206
|
+
* - action (give the action to approach the next goal)
|
|
207
|
+
* - act (execute the action)
|
|
208
|
+
* - loop
|
|
209
|
+
*
|
|
210
|
+
* ## Event System
|
|
211
|
+
* - `statuschange` - Agent status transitions (idle → running → completed/error/stopped)
|
|
212
|
+
* - `historychange` - History events updated (persistent, part of agent memory)
|
|
213
|
+
* - `activity` - Real-time activity feedback (transient, for UI only)
|
|
214
|
+
* - `dispose` - Agent cleanup triggered
|
|
215
|
+
*
|
|
216
|
+
* ## Information Streams
|
|
217
|
+
* 1. **History Events** (`history` array)
|
|
218
|
+
* - Persistent event stream that forms agent's memory
|
|
219
|
+
* - Included in LLM context across steps
|
|
220
|
+
* - Types: steps, observations, user takeovers, llm errors
|
|
221
|
+
*
|
|
222
|
+
* 2. **Activity Events** (via `activity` event)
|
|
223
|
+
* - Transient UI feedback during task execution
|
|
224
|
+
* - NOT included in LLM context
|
|
225
|
+
* - Types: thinking, executing, executed, retrying, error
|
|
226
|
+
*/
|
|
227
|
+
declare class AgentRuntime extends EventTarget {
|
|
228
|
+
#private;
|
|
229
|
+
readonly id: string;
|
|
230
|
+
readonly config: AgentRuntimeConfig & {
|
|
231
|
+
maxSteps: number;
|
|
232
|
+
};
|
|
233
|
+
readonly tools: typeof tools;
|
|
234
|
+
/** BrowserController for DOM operations */
|
|
235
|
+
readonly browserController: BrowserController;
|
|
236
|
+
task: string;
|
|
237
|
+
taskId: string;
|
|
238
|
+
/** History events */
|
|
239
|
+
history: HistoricalEvent[];
|
|
240
|
+
/** Whether this agent has been disposed */
|
|
241
|
+
disposed: boolean;
|
|
242
|
+
/**
|
|
243
|
+
* Called when the agent needs to ask the user questions.
|
|
244
|
+
* If unset, the `ask_user` tool will be disabled.
|
|
245
|
+
* Implementations should reject the promise when `signal` aborts.
|
|
246
|
+
* @example onAskUser: (q) => window.prompt(q) || ''
|
|
247
|
+
*/
|
|
248
|
+
onAskUser?: (question: string, options?: {
|
|
249
|
+
signal: AbortSignal;
|
|
250
|
+
}) => Promise<string>;
|
|
251
|
+
constructor(config: AgentRuntimeConfig);
|
|
252
|
+
/** Get current agent status */
|
|
253
|
+
get status(): AgentStatus;
|
|
254
|
+
/** Result of the most recent run, or `null` before the first run completes. */
|
|
255
|
+
get lastResult(): ExecutionResult | null;
|
|
256
|
+
/* Excluded from this release type: pushObservation */
|
|
257
|
+
/**
|
|
258
|
+
* Stop the current task and wait until the run has fully settled (including lifecycle hooks).
|
|
259
|
+
* @note never await .stop() in a lifecycle hook.
|
|
260
|
+
*/
|
|
261
|
+
stop(): Promise<void>;
|
|
262
|
+
/**
|
|
263
|
+
* external errors (pre-checks/config/hooks) will threw;
|
|
264
|
+
* agent errors will be caught and added to history, and return a failed result
|
|
265
|
+
*/
|
|
266
|
+
execute(task: string): Promise<ExecutionResult>;
|
|
267
|
+
dispose(): void;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
declare type AgentRuntimeConfig = AgentConfig & {
|
|
271
|
+
browserController: BrowserController;
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Agent lifecycle status.
|
|
276
|
+
*/
|
|
277
|
+
export declare type AgentStatus = 'idle' | 'running' | 'completed' | 'error' | 'stopped';
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* A single agent step with reflection and action
|
|
281
|
+
*/
|
|
282
|
+
declare interface AgentStepEvent {
|
|
283
|
+
type: 'step';
|
|
284
|
+
stepIndex: number;
|
|
285
|
+
reflection: Partial<AgentReflection>;
|
|
286
|
+
action: {
|
|
287
|
+
name: string;
|
|
288
|
+
input: any;
|
|
289
|
+
output: string;
|
|
290
|
+
};
|
|
291
|
+
usage: {
|
|
292
|
+
promptTokens: number;
|
|
293
|
+
completionTokens: number;
|
|
294
|
+
totalTokens: number;
|
|
295
|
+
cachedTokens?: number;
|
|
296
|
+
reasoningTokens?: number;
|
|
297
|
+
};
|
|
298
|
+
/** Raw LLM response for debugging */
|
|
299
|
+
rawResponse?: unknown;
|
|
300
|
+
/** Raw LLM request for debugging */
|
|
301
|
+
rawRequest?: unknown;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Internal tool definition that has access to PageAgent `this` context
|
|
306
|
+
*/
|
|
307
|
+
declare interface AgentTool<TParams = any> {
|
|
308
|
+
description: string;
|
|
309
|
+
inputSchema: z.ZodType<TParams>;
|
|
310
|
+
execute: (this: AgentRuntime, args: TParams, ctx: ToolContext) => Promise<string>;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* BrowserController manages DOM state and element interactions.
|
|
315
|
+
* It provides async methods for all DOM operations, keeping state isolated.
|
|
316
|
+
*
|
|
317
|
+
* @lifecycle
|
|
318
|
+
* - beforeUpdate: Emitted before the DOM tree is updated.
|
|
319
|
+
* - afterUpdate: Emitted after the DOM tree is updated.
|
|
320
|
+
*/
|
|
321
|
+
declare class BrowserController extends EventTarget {
|
|
322
|
+
private config;
|
|
323
|
+
/** Corresponds to eval_page in browser-use */
|
|
324
|
+
private flatTree;
|
|
325
|
+
/**
|
|
326
|
+
* All highlighted index-mapped interactive elements
|
|
327
|
+
* Corresponds to DOMState.selector_map in browser-use
|
|
328
|
+
*/
|
|
329
|
+
private selectorMap;
|
|
330
|
+
/** Index -> element text description mapping */
|
|
331
|
+
private elementTextMap;
|
|
332
|
+
/**
|
|
333
|
+
* Simplified HTML for LLM consumption.
|
|
334
|
+
* Corresponds to clickable_elements_to_string in browser-use
|
|
335
|
+
*/
|
|
336
|
+
private simplifiedHTML;
|
|
337
|
+
/** last time the tree was updated */
|
|
338
|
+
private lastTimeUpdate;
|
|
339
|
+
/** Whether the tree has been indexed at least once */
|
|
340
|
+
private isIndexed;
|
|
341
|
+
/** Visual mask overlay for blocking user interaction during automation */
|
|
342
|
+
private mask;
|
|
343
|
+
private maskReady;
|
|
344
|
+
private disposed;
|
|
345
|
+
constructor(config?: BrowserControllerConfig);
|
|
346
|
+
/**
|
|
347
|
+
* Initialize mask asynchronously (dynamic import to avoid CSS loading in Node)
|
|
348
|
+
*/
|
|
349
|
+
initMask(): void;
|
|
350
|
+
/**
|
|
351
|
+
* Get current page URL
|
|
352
|
+
*/
|
|
353
|
+
getCurrentUrl(): Promise<string>;
|
|
354
|
+
/**
|
|
355
|
+
* Get last tree update timestamp
|
|
356
|
+
*/
|
|
357
|
+
getLastUpdateTime(): Promise<number>;
|
|
358
|
+
/**
|
|
359
|
+
* Get structured browser state for LLM consumption.
|
|
360
|
+
* Automatically calls updateTree() to refresh the DOM state.
|
|
361
|
+
*/
|
|
362
|
+
getBrowserState(): Promise<BrowserState>;
|
|
363
|
+
/**
|
|
364
|
+
* Update DOM tree, returns simplified HTML for LLM.
|
|
365
|
+
* This is the main method to refresh the page state.
|
|
366
|
+
* Automatically bypasses mask during DOM extraction if enabled.
|
|
367
|
+
*/
|
|
368
|
+
updateTree(): Promise<string>;
|
|
369
|
+
/**
|
|
370
|
+
* Clean up all element highlights
|
|
371
|
+
*/
|
|
372
|
+
cleanUpHighlights(): Promise<void>;
|
|
373
|
+
/**
|
|
374
|
+
* Ensure the tree has been indexed before any index-based operation.
|
|
375
|
+
* Throws if updateTree() hasn't been called yet.
|
|
376
|
+
*/
|
|
377
|
+
private assertIndexed;
|
|
378
|
+
/**
|
|
379
|
+
* Click element by index
|
|
380
|
+
*/
|
|
381
|
+
clickElement(index: number): Promise<ActionResult>;
|
|
382
|
+
/**
|
|
383
|
+
* Input text into element by index
|
|
384
|
+
*/
|
|
385
|
+
inputText(index: number, text: string): Promise<ActionResult>;
|
|
386
|
+
/**
|
|
387
|
+
* Select dropdown option by index and option text
|
|
388
|
+
*/
|
|
389
|
+
selectOption(index: number, optionText: string): Promise<ActionResult>;
|
|
390
|
+
/**
|
|
391
|
+
* Scroll vertically
|
|
392
|
+
*/
|
|
393
|
+
scroll(options: {
|
|
394
|
+
down: boolean;
|
|
395
|
+
numPages: number;
|
|
396
|
+
pixels?: number;
|
|
397
|
+
index?: number;
|
|
398
|
+
}): Promise<ActionResult>;
|
|
399
|
+
/**
|
|
400
|
+
* Scroll horizontally
|
|
401
|
+
*/
|
|
402
|
+
scrollHorizontally(options: {
|
|
403
|
+
right: boolean;
|
|
404
|
+
pixels: number;
|
|
405
|
+
index?: number;
|
|
406
|
+
}): Promise<ActionResult>;
|
|
407
|
+
/**
|
|
408
|
+
* Execute arbitrary JavaScript on the page.
|
|
409
|
+
* The optional `signal` is exposed to the script scope so cooperative code
|
|
410
|
+
* can abort promptly when the task is stopped.
|
|
411
|
+
*/
|
|
412
|
+
executeJavascript(script: string, signal?: AbortSignal): Promise<ActionResult>;
|
|
413
|
+
/**
|
|
414
|
+
* Show the visual mask overlay.
|
|
415
|
+
* Only works after mask is setup.
|
|
416
|
+
*/
|
|
417
|
+
showMask(): Promise<void>;
|
|
418
|
+
/**
|
|
419
|
+
* Hide the visual mask overlay.
|
|
420
|
+
* Only works after mask is setup.
|
|
421
|
+
*/
|
|
422
|
+
hideMask(): Promise<void>;
|
|
423
|
+
/**
|
|
424
|
+
* Dispose and clean up resources
|
|
425
|
+
*/
|
|
426
|
+
dispose(): void;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Configuration for BrowserController
|
|
431
|
+
*/
|
|
432
|
+
declare interface BrowserControllerConfig extends dom.DomConfig {
|
|
433
|
+
/** Enable visual mask overlay during operations (default: false) */
|
|
434
|
+
enableMask?: boolean;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Structured browser state for LLM consumption
|
|
439
|
+
*/
|
|
440
|
+
declare interface BrowserState {
|
|
441
|
+
url: string;
|
|
442
|
+
title: string;
|
|
443
|
+
/** Page info + scroll position hint (e.g. "Page info: 1920x1080px...\n[Start of page]") */
|
|
444
|
+
header: string;
|
|
445
|
+
/** Simplified HTML of interactive elements */
|
|
446
|
+
content: string;
|
|
447
|
+
/** Page footer hint (e.g. "... 300 pixels below ..." or "[End of page]") */
|
|
448
|
+
footer: string;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
declare function cleanUpHighlights(): void;
|
|
452
|
+
|
|
453
|
+
export declare function createAgent(config: MyPageAgentConfig): MyPageAgent;
|
|
454
|
+
|
|
455
|
+
declare namespace dom {
|
|
456
|
+
export {
|
|
457
|
+
resolveViewportExpansion,
|
|
458
|
+
getFlatTree,
|
|
459
|
+
flatTreeToString,
|
|
460
|
+
getSelectorMap,
|
|
461
|
+
getElementTextMap,
|
|
462
|
+
cleanUpHighlights,
|
|
463
|
+
BrowserState,
|
|
464
|
+
DomConfig,
|
|
465
|
+
getAllTextTillNextClickableElement
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
declare interface DomConfig {
|
|
470
|
+
viewportExpansion?: number;
|
|
471
|
+
interactiveBlacklist?: (Element | (() => Element))[];
|
|
472
|
+
interactiveWhitelist?: (Element | (() => Element))[];
|
|
473
|
+
includeAttributes?: string[];
|
|
474
|
+
highlightOpacity?: number;
|
|
475
|
+
highlightLabelOpacity?: number;
|
|
476
|
+
/**
|
|
477
|
+
* Preserve semantic landmark tags in dehydrated output even if not interactive
|
|
478
|
+
* @note maybe confusing for LLM combining with page scrolling, use with caution
|
|
479
|
+
**/
|
|
480
|
+
keepSemanticTags?: boolean;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
declare type DomNode = TextDomNode | ElementDomNode | InteractiveElementDomNode;
|
|
484
|
+
|
|
485
|
+
declare interface ElementDomNode {
|
|
486
|
+
tagName: string;
|
|
487
|
+
attributes?: Record<string, string>;
|
|
488
|
+
xpath?: string;
|
|
489
|
+
children?: string[];
|
|
490
|
+
isVisible?: boolean;
|
|
491
|
+
isTopElement?: boolean;
|
|
492
|
+
isInViewport?: boolean;
|
|
493
|
+
isNew?: boolean;
|
|
494
|
+
isInteractive?: false;
|
|
495
|
+
highlightIndex?: number;
|
|
496
|
+
extra?: Record<string, any>;
|
|
497
|
+
[key: string]: unknown;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
export declare interface ExecutionResult {
|
|
501
|
+
success: boolean;
|
|
502
|
+
data: string;
|
|
503
|
+
history: HistoricalEvent[];
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
declare interface FlatDomTree {
|
|
507
|
+
rootId: string;
|
|
508
|
+
map: Record<string, DomNode>;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* 对应 python 中的 views::clickable_elements_to_string,
|
|
513
|
+
* 将 dom 信息处理成适合 llm 阅读的文本格式
|
|
514
|
+
* @形如
|
|
515
|
+
* ``` text
|
|
516
|
+
* [0]<a aria-label=page-agent.js 首页 />
|
|
517
|
+
* [1]<div >P />
|
|
518
|
+
* [2]<div >page-agent.js
|
|
519
|
+
* UI Agent in your webpage />
|
|
520
|
+
* [3]<a >文档 />
|
|
521
|
+
* [4]<a aria-label=查看源码(在新窗口打开)>源码 />
|
|
522
|
+
* UI Agent in your webpage
|
|
523
|
+
* 用户输入需求,AI 理解页面并自动操作。
|
|
524
|
+
* [5]<a role=button>快速开始 />
|
|
525
|
+
* [6]<a role=button>查看文档 />
|
|
526
|
+
* 无需后端
|
|
527
|
+
* ```
|
|
528
|
+
* 其中可交互元素用序号标出,提示llm可以用序号操作。
|
|
529
|
+
* 缩进代表父子关系。
|
|
530
|
+
* 普通文本则直接列出来。
|
|
531
|
+
*
|
|
532
|
+
* @todo 数据脱敏过滤器
|
|
533
|
+
*/
|
|
534
|
+
declare function flatTreeToString(flatTree: FlatDomTree, includeAttributes?: string[], keepSemanticTags?: boolean): string;
|
|
535
|
+
|
|
536
|
+
declare const getAllTextTillNextClickableElement: (node: TreeNode, maxDepth?: number) => string;
|
|
537
|
+
|
|
538
|
+
declare function getElementTextMap(simplifiedHTML: string): Map<number, string>;
|
|
539
|
+
|
|
540
|
+
declare function getFlatTree(config: DomConfig): FlatDomTree;
|
|
541
|
+
|
|
542
|
+
declare function getSelectorMap(flatTree: FlatDomTree): Map<number, InteractiveElementDomNode>;
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Union type for all history events
|
|
546
|
+
*/
|
|
547
|
+
export declare type HistoricalEvent = AgentStepEvent | ObservationEvent | UserTakeoverEvent | RetryEvent | AgentErrorEvent;
|
|
548
|
+
|
|
549
|
+
declare interface InteractiveElementDomNode {
|
|
550
|
+
tagName: string;
|
|
551
|
+
attributes?: Record<string, string>;
|
|
552
|
+
xpath?: string;
|
|
553
|
+
children?: string[];
|
|
554
|
+
isVisible?: boolean;
|
|
555
|
+
isTopElement?: boolean;
|
|
556
|
+
isInViewport?: boolean;
|
|
557
|
+
isInteractive: true;
|
|
558
|
+
highlightIndex: number;
|
|
559
|
+
/**
|
|
560
|
+
* 可交互元素的 dom 引用
|
|
561
|
+
*/
|
|
562
|
+
ref: HTMLElement;
|
|
563
|
+
[key: string]: unknown;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/**
|
|
567
|
+
* LLM configuration
|
|
568
|
+
*/
|
|
569
|
+
declare interface LLMConfig {
|
|
570
|
+
baseURL: string;
|
|
571
|
+
model: string;
|
|
572
|
+
apiKey?: string;
|
|
573
|
+
/**
|
|
574
|
+
* @deprecated No longer a standard parameter; many models reject it outright.
|
|
575
|
+
* Use `transformRequestBody` to set it only for models you've verified.
|
|
576
|
+
*/
|
|
577
|
+
temperature?: number;
|
|
578
|
+
maxRetries?: number;
|
|
579
|
+
/**
|
|
580
|
+
* Transform the final request body before sending it to the provider.
|
|
581
|
+
* Use this to implement provider-specific request tweaks such as caching hints or custom flags.
|
|
582
|
+
*
|
|
583
|
+
* Return a new object, or mutate the input object and return undefined.
|
|
584
|
+
*/
|
|
585
|
+
transformRequestBody?: (requestBody: Record<string, unknown>) => Record<string, unknown> | undefined;
|
|
586
|
+
/**
|
|
587
|
+
* remove the tool_choice field from the request.
|
|
588
|
+
* @note fix "Invalid tool_choice type: 'object'" for some LLMs.
|
|
589
|
+
*/
|
|
590
|
+
disableNamedToolChoice?: boolean;
|
|
591
|
+
/**
|
|
592
|
+
* Custom fetch function for LLM API requests.
|
|
593
|
+
* Use this to customize headers, credentials, proxy, etc.
|
|
594
|
+
* The response should follow OpenAI API format.
|
|
595
|
+
*/
|
|
596
|
+
customFetch?: typeof globalThis.fetch;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
declare const locales: {
|
|
600
|
+
readonly 'en-US': {
|
|
601
|
+
readonly ui: {
|
|
602
|
+
readonly ready: "Ready";
|
|
603
|
+
readonly thinking: "Thinking...";
|
|
604
|
+
readonly taskInput: "Enter new task, describe steps in detail, press Enter to submit";
|
|
605
|
+
readonly userAnswerPrompt: "Please answer the question above, press Enter to submit";
|
|
606
|
+
readonly taskTerminated: "Task terminated";
|
|
607
|
+
readonly taskCompleted: "Task completed";
|
|
608
|
+
readonly userAnswer: "User answer: {{input}}";
|
|
609
|
+
readonly question: "Question: {{question}}";
|
|
610
|
+
readonly waitingPlaceholder: "Waiting for task to start...";
|
|
611
|
+
readonly stop: "Stop";
|
|
612
|
+
readonly close: "Close";
|
|
613
|
+
readonly expand: "Expand history";
|
|
614
|
+
readonly collapse: "Collapse history";
|
|
615
|
+
readonly step: "Step {{number}}";
|
|
616
|
+
readonly tools: {
|
|
617
|
+
readonly clicking: "Clicking element [{{index}}]...";
|
|
618
|
+
readonly inputting: "Inputting text to element [{{index}}]...";
|
|
619
|
+
readonly selecting: "Selecting option \"{{text}}\"...";
|
|
620
|
+
readonly scrolling: "Scrolling page...";
|
|
621
|
+
readonly waiting: "Waiting {{seconds}} seconds...";
|
|
622
|
+
readonly askingUser: "Asking user...";
|
|
623
|
+
readonly done: "Task done";
|
|
624
|
+
readonly clicked: "🖱️ Clicked element [{{index}}]";
|
|
625
|
+
readonly inputted: "⌨️ Inputted text \"{{text}}\"";
|
|
626
|
+
readonly selected: "☑️ Selected option \"{{text}}\"";
|
|
627
|
+
readonly scrolled: "🛞 Page scrolled";
|
|
628
|
+
readonly waited: "⌛️ Wait completed";
|
|
629
|
+
readonly executing: "Executing {{toolName}}...";
|
|
630
|
+
readonly resultSuccess: "success";
|
|
631
|
+
readonly resultFailure: "failed";
|
|
632
|
+
readonly resultError: "error";
|
|
633
|
+
};
|
|
634
|
+
readonly errors: {
|
|
635
|
+
readonly elementNotFound: "No interactive element found at index {{index}}";
|
|
636
|
+
readonly taskRequired: "Task description is required";
|
|
637
|
+
readonly executionFailed: "Task execution failed";
|
|
638
|
+
readonly notInputElement: "Element is not an input or textarea";
|
|
639
|
+
readonly notSelectElement: "Element is not a select element";
|
|
640
|
+
readonly optionNotFound: "Option \"{{text}}\" not found";
|
|
641
|
+
};
|
|
642
|
+
};
|
|
643
|
+
};
|
|
644
|
+
readonly 'zh-CN': {
|
|
645
|
+
readonly ui: {
|
|
646
|
+
readonly ready: "准备就绪";
|
|
647
|
+
readonly thinking: "正在思考...";
|
|
648
|
+
readonly taskInput: "输入新任务,详细描述步骤,回车提交";
|
|
649
|
+
readonly userAnswerPrompt: "请回答上面问题,回车提交";
|
|
650
|
+
readonly taskTerminated: "任务已终止";
|
|
651
|
+
readonly taskCompleted: "任务结束";
|
|
652
|
+
readonly userAnswer: "用户回答: {{input}}";
|
|
653
|
+
readonly question: "询问: {{question}}";
|
|
654
|
+
readonly waitingPlaceholder: "等待任务开始...";
|
|
655
|
+
readonly stop: "终止";
|
|
656
|
+
readonly close: "关闭";
|
|
657
|
+
readonly expand: "展开历史";
|
|
658
|
+
readonly collapse: "收起历史";
|
|
659
|
+
readonly step: "步骤 {{number}}";
|
|
660
|
+
readonly tools: {
|
|
661
|
+
readonly clicking: "正在点击元素 [{{index}}]...";
|
|
662
|
+
readonly inputting: "正在输入文本到元素 [{{index}}]...";
|
|
663
|
+
readonly selecting: "正在选择选项 \"{{text}}\"...";
|
|
664
|
+
readonly scrolling: "正在滚动页面...";
|
|
665
|
+
readonly waiting: "等待 {{seconds}} 秒...";
|
|
666
|
+
readonly askingUser: "正在询问用户...";
|
|
667
|
+
readonly done: "结束任务";
|
|
668
|
+
readonly clicked: "🖱️ 已点击元素 [{{index}}]";
|
|
669
|
+
readonly inputted: "⌨️ 已输入文本 \"{{text}}\"";
|
|
670
|
+
readonly selected: "☑️ 已选择选项 \"{{text}}\"";
|
|
671
|
+
readonly scrolled: "🛞 页面滚动完成";
|
|
672
|
+
readonly waited: "⌛️ 等待完成";
|
|
673
|
+
readonly executing: "正在执行 {{toolName}}...";
|
|
674
|
+
readonly resultSuccess: "成功";
|
|
675
|
+
readonly resultFailure: "失败";
|
|
676
|
+
readonly resultError: "错误";
|
|
677
|
+
};
|
|
678
|
+
readonly errors: {
|
|
679
|
+
readonly elementNotFound: "未找到索引为 {{index}} 的交互元素";
|
|
680
|
+
readonly taskRequired: "任务描述不能为空";
|
|
681
|
+
readonly executionFailed: "任务执行失败";
|
|
682
|
+
readonly notInputElement: "元素不是输入框或文本域";
|
|
683
|
+
readonly notSelectElement: "元素不是选择框";
|
|
684
|
+
readonly optionNotFound: "未找到选项 \"{{text}}\"";
|
|
685
|
+
};
|
|
686
|
+
};
|
|
687
|
+
};
|
|
688
|
+
};
|
|
689
|
+
|
|
690
|
+
export declare class MyPageAgent extends AgentRuntime {
|
|
691
|
+
readonly ui: UI;
|
|
692
|
+
constructor(config: MyPageAgentConfig);
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
export declare type MyPageAgentConfig = Omit<AgentConfig, 'experimentalScriptExecutionTool'> & BrowserControllerConfig & Omit<UIConfig, 'language'>;
|
|
696
|
+
|
|
697
|
+
/**
|
|
698
|
+
* Persistent observation event (stays in memory)
|
|
699
|
+
*/
|
|
700
|
+
declare interface ObservationEvent {
|
|
701
|
+
type: 'observation';
|
|
702
|
+
content: string;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
declare function resolveViewportExpansion(viewportExpansion?: number): number;
|
|
706
|
+
|
|
707
|
+
/**
|
|
708
|
+
* Retry event - LLM call is being retried
|
|
709
|
+
*/
|
|
710
|
+
declare interface RetryEvent {
|
|
711
|
+
type: 'retry';
|
|
712
|
+
message: string;
|
|
713
|
+
attempt: number;
|
|
714
|
+
maxAttempts: number;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/** Supported UI languages */
|
|
718
|
+
declare type SupportedLanguage = 'en-US' | 'zh-CN';
|
|
719
|
+
|
|
720
|
+
declare type SupportedLanguage_2 = keyof typeof locales;
|
|
721
|
+
|
|
722
|
+
declare interface TextDomNode {
|
|
723
|
+
type: 'TEXT_NODE';
|
|
724
|
+
text: string;
|
|
725
|
+
isVisible: boolean;
|
|
726
|
+
[key: string]: unknown;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
export declare function tool<TParams>(options: AgentTool<TParams>): AgentTool<TParams>;
|
|
730
|
+
|
|
731
|
+
/**
|
|
732
|
+
* Per-invocation context passed to every tool execution.
|
|
733
|
+
* Tools MUST honor `signal` to support cooperative cancellation.
|
|
734
|
+
*/
|
|
735
|
+
declare interface ToolContext {
|
|
736
|
+
signal: AbortSignal;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
/**
|
|
740
|
+
* Internal tools for PageAgent.
|
|
741
|
+
* Note: Using any to allow different parameter types for each tool
|
|
742
|
+
*/
|
|
743
|
+
declare const tools: Map<string, AgentTool<any>>;
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* elementsToString 内部使用的类型
|
|
747
|
+
*/
|
|
748
|
+
declare interface TreeNode {
|
|
749
|
+
type: 'text' | 'element';
|
|
750
|
+
parent: TreeNode | null;
|
|
751
|
+
children: TreeNode[];
|
|
752
|
+
isVisible: boolean;
|
|
753
|
+
text?: string;
|
|
754
|
+
tagName?: string;
|
|
755
|
+
attributes?: Record<string, string>;
|
|
756
|
+
isInteractive?: boolean;
|
|
757
|
+
isTopElement?: boolean;
|
|
758
|
+
isNew?: boolean;
|
|
759
|
+
highlightIndex?: number;
|
|
760
|
+
extra?: Record<string, any>;
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
/**
|
|
764
|
+
* Agent control UI
|
|
765
|
+
*
|
|
766
|
+
* Architecture:
|
|
767
|
+
* - History list: renders directly from agent.history (historical events)
|
|
768
|
+
* - Header bar: shows activity events (transient state) and agent status
|
|
769
|
+
*
|
|
770
|
+
* This separation ensures data consistency - history is the single source of truth
|
|
771
|
+
* for what has been done, while activity shows what is happening now.
|
|
772
|
+
*/
|
|
773
|
+
declare class UI {
|
|
774
|
+
#private;
|
|
775
|
+
get wrapper(): HTMLElement;
|
|
776
|
+
/**
|
|
777
|
+
* Create a UI bound to an agent
|
|
778
|
+
* @param agent - Agent instance that implements UIAdapter
|
|
779
|
+
* @param config - Optional UI configuration
|
|
780
|
+
*/
|
|
781
|
+
constructor(agent: UIAdapter, config?: UIConfig);
|
|
782
|
+
show(): void;
|
|
783
|
+
hide(): void;
|
|
784
|
+
reset(): void;
|
|
785
|
+
expand(): void;
|
|
786
|
+
collapse(): void;
|
|
787
|
+
/**
|
|
788
|
+
* Dispose UI and clean up event listeners
|
|
789
|
+
*/
|
|
790
|
+
dispose(): void;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
/**
|
|
794
|
+
* Minimal interface that UI expects from an agent.
|
|
795
|
+
* UI does not depend on PageAgent directly - it only requires this interface.
|
|
796
|
+
* This enables decoupling and allows any agent implementation to work with UI.
|
|
797
|
+
*
|
|
798
|
+
* Events:
|
|
799
|
+
* - 'statuschange': Agent status changed
|
|
800
|
+
* - 'historychange': Historical events updated (persisted)
|
|
801
|
+
* - 'activity': Transient activity for immediate UI feedback (thinking/executing/etc)
|
|
802
|
+
* - 'dispose': Agent is being disposed
|
|
803
|
+
*/
|
|
804
|
+
declare interface UIAdapter extends EventTarget {
|
|
805
|
+
/** Current agent status */
|
|
806
|
+
readonly status: 'idle' | 'running' | 'completed' | 'error' | 'stopped';
|
|
807
|
+
/** Result of the most recent run, or `null` before the first run completes */
|
|
808
|
+
readonly lastResult: {
|
|
809
|
+
success: boolean;
|
|
810
|
+
} | null;
|
|
811
|
+
/** History of agent events */
|
|
812
|
+
readonly history: readonly {
|
|
813
|
+
type: 'step' | 'observation' | 'user_takeover' | 'retry' | 'error';
|
|
814
|
+
stepIndex?: number;
|
|
815
|
+
/** For 'step' type */
|
|
816
|
+
reflection?: {
|
|
817
|
+
evaluation_previous_goal?: string;
|
|
818
|
+
memory?: string;
|
|
819
|
+
next_goal?: string;
|
|
820
|
+
};
|
|
821
|
+
/** For 'step' type */
|
|
822
|
+
action?: {
|
|
823
|
+
name: string;
|
|
824
|
+
input: unknown;
|
|
825
|
+
output: string;
|
|
826
|
+
};
|
|
827
|
+
/** For 'observation' type */
|
|
828
|
+
content?: string;
|
|
829
|
+
/** For 'retry' type */
|
|
830
|
+
attempt?: number;
|
|
831
|
+
maxAttempts?: number;
|
|
832
|
+
/** For 'retry' and 'error' types */
|
|
833
|
+
message?: string;
|
|
834
|
+
}[];
|
|
835
|
+
/** Current task being executed */
|
|
836
|
+
readonly task: string;
|
|
837
|
+
/**
|
|
838
|
+
* Called when the agent needs to ask the user questions.
|
|
839
|
+
* If unset, the `ask_user` tool will be disabled.
|
|
840
|
+
* UI will set this to handle user questions via its UI.
|
|
841
|
+
* The optional `signal` aborts when the task is stopped or disposed.
|
|
842
|
+
*/
|
|
843
|
+
onAskUser?: (question: string, options?: {
|
|
844
|
+
signal: AbortSignal;
|
|
845
|
+
}) => Promise<string>;
|
|
846
|
+
/** Execute a task */
|
|
847
|
+
execute(task: string): Promise<unknown>;
|
|
848
|
+
/** Stop the current task (agent remains reusable) */
|
|
849
|
+
stop(): Promise<void>;
|
|
850
|
+
/** Dispose the agent (terminal, cannot be reused) */
|
|
851
|
+
dispose(): void;
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/**
|
|
855
|
+
* UI configuration
|
|
856
|
+
*/
|
|
857
|
+
declare interface UIConfig {
|
|
858
|
+
language?: SupportedLanguage_2;
|
|
859
|
+
/**
|
|
860
|
+
* Whether to prompt for next task after task completion
|
|
861
|
+
* @default true
|
|
862
|
+
*/
|
|
863
|
+
promptForNextTask?: boolean;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* User takeover event
|
|
868
|
+
*/
|
|
869
|
+
declare interface UserTakeoverEvent {
|
|
870
|
+
type: 'user_takeover';
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
export { }
|