@agentium/browser 3.1.0 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/action-space.d.ts +42 -0
- package/dist/action-space.d.ts.map +1 -0
- package/dist/browser-agent.d.ts +131 -0
- package/dist/browser-agent.d.ts.map +1 -0
- package/dist/browser-provider.d.ts +242 -0
- package/dist/browser-provider.d.ts.map +1 -0
- package/dist/credential-vault.d.ts +37 -0
- package/dist/credential-vault.d.ts.map +1 -0
- package/dist/index.cjs +512 -31
- package/dist/index.d.ts +7 -801
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +502 -31
- package/dist/loop-detector.d.ts +83 -0
- package/dist/loop-detector.d.ts.map +1 -0
- package/dist/prompts.d.ts +21 -0
- package/dist/prompts.d.ts.map +1 -0
- package/dist/stealth.d.ts +27 -0
- package/dist/stealth.d.ts.map +1 -0
- package/dist/{index.d.cts → types.d.ts} +64 -387
- package/dist/types.d.ts.map +1 -0
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -21,7 +21,17 @@ const agent = new BrowserAgent({
|
|
|
21
21
|
});
|
|
22
22
|
|
|
23
23
|
const result = await agent.run("Go to example.com and get the page title");
|
|
24
|
-
console.log(result.
|
|
24
|
+
console.log(result.result);
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Jev can pick the next click from a closed per-step list (`planner: "jev"`). Do not set `model: jev()`.
|
|
28
|
+
|
|
29
|
+
```typescript
|
|
30
|
+
const agent = new BrowserAgent({
|
|
31
|
+
name: "jev-browser",
|
|
32
|
+
model: openai("gpt-4o-mini"),
|
|
33
|
+
planner: "jev",
|
|
34
|
+
});
|
|
25
35
|
```
|
|
26
36
|
|
|
27
37
|
## Documentation
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { BrowserAction, DomElement, SearchEngine } from "./types.js";
|
|
2
|
+
export declare const DEFAULT_MAX_ACTION_CHOICES = 40;
|
|
3
|
+
export interface TabInfo {
|
|
4
|
+
id: string;
|
|
5
|
+
url: string;
|
|
6
|
+
active: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface ActionSpace {
|
|
9
|
+
/** Jev / TypeSafe choice criteria: label → description. */
|
|
10
|
+
criteria: Record<string, string>;
|
|
11
|
+
toAction(label: string): BrowserAction | undefined;
|
|
12
|
+
}
|
|
13
|
+
export declare function searchUrl(query: string, engine?: SearchEngine): string;
|
|
14
|
+
export declare function isSearchResultsUrl(url: string): boolean;
|
|
15
|
+
/** Bot-block / challenge pages (DDG 418, Google sorry, Cloudflare, …). */
|
|
16
|
+
export declare function isBlockedPageUrl(url: string): boolean;
|
|
17
|
+
export declare function isBotChallengeText(text: string): boolean;
|
|
18
|
+
export declare function isResultTitle(label: string): boolean;
|
|
19
|
+
export declare function looksLikeResultList(text: string): boolean;
|
|
20
|
+
/** Longest non-chrome link labels — search result titles, not "email us". */
|
|
21
|
+
export declare function titlesFromElements(elements: DomElement[], max?: number): string;
|
|
22
|
+
export declare const SERP_TITLE_SELECTORS: string[];
|
|
23
|
+
export declare function guessSearchQuery(task: string): string;
|
|
24
|
+
/**
|
|
25
|
+
* Closed action list for this frame: chrome (back, scroll, done, …) plus
|
|
26
|
+
* `click_N` / `type_N` from the DOM snapshot. Used by the Jev planner and
|
|
27
|
+
* as documentation of what `executeAction` can run without free-form JSON.
|
|
28
|
+
*/
|
|
29
|
+
export declare function buildActionSpace(elements: DomElement[], tabs?: TabInfo[], opts?: {
|
|
30
|
+
max?: number;
|
|
31
|
+
pagesBelow?: number;
|
|
32
|
+
pagesAbove?: number;
|
|
33
|
+
allowSearch?: boolean;
|
|
34
|
+
allowDone?: boolean;
|
|
35
|
+
allowWait?: boolean;
|
|
36
|
+
}): ActionSpace;
|
|
37
|
+
export declare function labelToAction(label: string, extras?: {
|
|
38
|
+
typeText?: string;
|
|
39
|
+
searchQuery?: string;
|
|
40
|
+
doneResult?: string;
|
|
41
|
+
}): BrowserAction | undefined;
|
|
42
|
+
//# sourceMappingURL=action-space.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"action-space.d.ts","sourceRoot":"","sources":["../src/action-space.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1E,eAAO,MAAM,0BAA0B,KAAK,CAAC;AAE7C,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAAC;CACpD;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,YAA2B,GAAG,MAAM,CAWpF;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAavD;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAErD;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAIxD;AAKD,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAMpD;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAMzD;AAED,6EAA6E;AAC7E,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,GAAG,SAAI,GAAG,MAAM,CAe1E;AAED,eAAO,MAAM,oBAAoB,UAAkF,CAAC;AAEpH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOrD;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,UAAU,EAAE,EACtB,IAAI,GAAE,OAAO,EAAO,EACpB,IAAI,CAAC,EAAE;IACL,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,GACA,WAAW,CAgDb;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GACxE,aAAa,GAAG,SAAS,CAyB3B"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import type { ToolDef } from "@agentium/core";
|
|
2
|
+
import { EventBus, MemoryManager } from "@agentium/core";
|
|
3
|
+
import type { BrowserAgentConfig, BrowserRunOpts, BrowserRunOutput } from "./types.js";
|
|
4
|
+
export declare class BrowserAgent {
|
|
5
|
+
readonly name: string;
|
|
6
|
+
readonly eventBus: EventBus;
|
|
7
|
+
private model;
|
|
8
|
+
private pageExtractionLLM;
|
|
9
|
+
private fallbackModel;
|
|
10
|
+
private useThinking;
|
|
11
|
+
private historyWindow;
|
|
12
|
+
private instructions?;
|
|
13
|
+
private extendSystemMessage?;
|
|
14
|
+
private overrideSystemMessage?;
|
|
15
|
+
private maxSteps;
|
|
16
|
+
private maxFailures;
|
|
17
|
+
private maxActionsPerStep;
|
|
18
|
+
private initialActions;
|
|
19
|
+
private useVision;
|
|
20
|
+
private directlyOpenUrl;
|
|
21
|
+
private headless;
|
|
22
|
+
private viewport;
|
|
23
|
+
private defaultStartUrl?;
|
|
24
|
+
private waitAfterAction;
|
|
25
|
+
private maxRepeats;
|
|
26
|
+
private useDOM;
|
|
27
|
+
private allowEvaluate;
|
|
28
|
+
private allowedDomains?;
|
|
29
|
+
private prohibitedDomains?;
|
|
30
|
+
private storageState?;
|
|
31
|
+
private cdpUrl?;
|
|
32
|
+
private recordVideo?;
|
|
33
|
+
private credentials?;
|
|
34
|
+
private stealth?;
|
|
35
|
+
private humanize?;
|
|
36
|
+
private tools;
|
|
37
|
+
private planner;
|
|
38
|
+
private jevModel;
|
|
39
|
+
private maxActionChoices;
|
|
40
|
+
private searchEngine;
|
|
41
|
+
private jevProvider;
|
|
42
|
+
private costTracker;
|
|
43
|
+
private memoryManager;
|
|
44
|
+
private logger;
|
|
45
|
+
/** Access the MemoryManager (if memory is configured). */
|
|
46
|
+
get memory(): MemoryManager | null;
|
|
47
|
+
constructor(config: BrowserAgentConfig);
|
|
48
|
+
run(task: string, opts?: BrowserRunOpts): Promise<BrowserRunOutput>;
|
|
49
|
+
/**
|
|
50
|
+
* Returns a ToolDef that lets a regular Agent delegate browser tasks
|
|
51
|
+
* to this BrowserAgent.
|
|
52
|
+
*/
|
|
53
|
+
asTool(config?: {
|
|
54
|
+
name?: string;
|
|
55
|
+
description?: string;
|
|
56
|
+
}): ToolDef;
|
|
57
|
+
private shouldCaptureVision;
|
|
58
|
+
private detectUrlInTask;
|
|
59
|
+
private assertDomainAllowed;
|
|
60
|
+
/**
|
|
61
|
+
* Build the message array sent to the model: system prompt + a compact
|
|
62
|
+
* summary of older turns (if any) + the most recent `historyWindow`
|
|
63
|
+
* turns verbatim + the current step's user message.
|
|
64
|
+
*
|
|
65
|
+
* Inspired by browser-use's history compaction. Keeps tokens bounded
|
|
66
|
+
* while giving the model meaningful context about what it already
|
|
67
|
+
* tried.
|
|
68
|
+
*/
|
|
69
|
+
private buildMessages;
|
|
70
|
+
/**
|
|
71
|
+
* Call the primary model, retrying once with `fallbackModel` on
|
|
72
|
+
* transient errors (5xx, 429, network). Returns the response or
|
|
73
|
+
* `null` if both models failed.
|
|
74
|
+
*/
|
|
75
|
+
private callModelWithFallback;
|
|
76
|
+
private isTransientError;
|
|
77
|
+
private getJev;
|
|
78
|
+
/**
|
|
79
|
+
* Ask Jev to pick one label from this frame's action space, then map it
|
|
80
|
+
* to a BrowserAction. Type/search strings come from a text model or the task.
|
|
81
|
+
*/
|
|
82
|
+
private planWithJev;
|
|
83
|
+
private pageLooksBlocked;
|
|
84
|
+
private scrapeSerpTitles;
|
|
85
|
+
private inferTypeText;
|
|
86
|
+
/**
|
|
87
|
+
* Parse the model's raw response into an `AgentOutput`. Tolerant to
|
|
88
|
+
* three shapes:
|
|
89
|
+
* - Full envelope: { thinking, evaluation_previous_goal, action, ... }
|
|
90
|
+
* - Raw action object (legacy / `useThinking: false`)
|
|
91
|
+
* - Raw action array
|
|
92
|
+
*
|
|
93
|
+
* Also strips ```json fences the model occasionally adds.
|
|
94
|
+
*/
|
|
95
|
+
private parseEnvelope;
|
|
96
|
+
/**
|
|
97
|
+
* Quick post-navigation health check. If the page came back blank
|
|
98
|
+
* (no body text and no interactive elements), reload once and wait
|
|
99
|
+
* for stable. This catches the "FreightOS half-loaded font test" class
|
|
100
|
+
* of failure before the LLM ever sees it.
|
|
101
|
+
*/
|
|
102
|
+
private navigationHealthCheck;
|
|
103
|
+
/**
|
|
104
|
+
* Force-finalize the run with whatever partial data the agent has.
|
|
105
|
+
* Called when:
|
|
106
|
+
* - `maxSteps` is exhausted without an explicit `done`,
|
|
107
|
+
* - `maxFailures` is exceeded,
|
|
108
|
+
* - the model can't produce parseable JSON enough times to make
|
|
109
|
+
* forward progress.
|
|
110
|
+
* The result is composed from `extractedContent` + the last few
|
|
111
|
+
* action summaries so the caller gets something useful instead of
|
|
112
|
+
* just a one-line error.
|
|
113
|
+
*/
|
|
114
|
+
private forceDone;
|
|
115
|
+
private finalize;
|
|
116
|
+
/**
|
|
117
|
+
* Execute a single action. Returns `{ output?, didNavigate? }` for the
|
|
118
|
+
* caller's state tracking. May throw — the loop handles failure-budget
|
|
119
|
+
* accounting in that case.
|
|
120
|
+
*/
|
|
121
|
+
private executeAction;
|
|
122
|
+
private sleep;
|
|
123
|
+
/**
|
|
124
|
+
* Parse a quoted target keyword from a click action's `description`.
|
|
125
|
+
* Returns `undefined` for generic / ambiguous labels (login buttons,
|
|
126
|
+
* close, OK, etc.) where a substring text match could fire on the
|
|
127
|
+
* wrong element.
|
|
128
|
+
*/
|
|
129
|
+
private extractClickKeyword;
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=browser-agent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser-agent.d.ts","sourceRoot":"","sources":["../src/browser-agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAuD,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACnG,OAAO,EAAU,QAAQ,EAAe,aAAa,EAAc,MAAM,gBAAgB,CAAC;AAoB1F,OAAO,KAAK,EAGV,kBAAkB,EAElB,cAAc,EACd,gBAAgB,EAKjB,MAAM,YAAY,CAAC;AAEpB,qBAAa,YAAY;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAE5B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAC,CAAS;IACrC,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,SAAS,CAAmB;IACpC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAoC;IACpD,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,cAAc,CAAC,CAAW;IAClC,OAAO,CAAC,iBAAiB,CAAC,CAAW;IACrC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,WAAW,CAAC,CAA4B;IAChD,OAAO,CAAC,WAAW,CAAC,CAAkB;IACtC,OAAO,CAAC,OAAO,CAAC,CAA+C;IAC/D,OAAO,CAAC,QAAQ,CAAC,CAAgD;IACjE,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,MAAM,CAAS;IAEvB,0DAA0D;IAC1D,IAAI,MAAM,IAAI,aAAa,GAAG,IAAI,CAEjC;gBAEW,MAAM,EAAE,kBAAkB;IAgDhC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkWzE;;;OAGG;IACH,MAAM,CAAC,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO;IAmBjE,OAAO,CAAC,mBAAmB;IAa3B,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,mBAAmB;IAgB3B;;;;;;;;OAQG;IACH,OAAO,CAAC,aAAa;IA2CrB;;;;OAIG;YACW,qBAAqB;IA4BnC,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,MAAM;IAKd;;;OAGG;YACW,WAAW;YAoFX,gBAAgB;YAUhB,gBAAgB;YAqBhB,aAAa;IAiB3B;;;;;;;;OAQG;IACH,OAAO,CAAC,aAAa;IAoCrB;;;;;OAKG;YACW,qBAAqB;IAiBnC;;;;;;;;;;OAUG;YACW,SAAS;YAwBT,QAAQ;IA6EtB;;;;OAIG;YACW,aAAa;IA0N3B,OAAO,CAAC,KAAK;IAIb;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;CAmC5B"}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import type { DomElement, DomSnapshot, HumanizeConfig, PageInfo, StealthConfig } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Playwright wrapper with stealth anti-detection, human-like behavior,
|
|
4
|
+
* indexed DOM element resolution, and a rich action vocabulary.
|
|
5
|
+
*
|
|
6
|
+
* The `BrowserProvider` is intentionally LLM-agnostic — it exposes the
|
|
7
|
+
* primitives that `BrowserAgent` orchestrates via vision+DOM reasoning.
|
|
8
|
+
*/
|
|
9
|
+
export declare class BrowserProvider {
|
|
10
|
+
private browser;
|
|
11
|
+
private context;
|
|
12
|
+
private page;
|
|
13
|
+
private pages;
|
|
14
|
+
private activeTabId;
|
|
15
|
+
private tabCounter;
|
|
16
|
+
private _viewport;
|
|
17
|
+
private _videoDir?;
|
|
18
|
+
private _humanize?;
|
|
19
|
+
/**
|
|
20
|
+
* Most recent DOM snapshot (one per `extractDOM` call). Indexed actions
|
|
21
|
+
* (`clickByIndex`, `inputByIndex`, …) resolve their `index` against this.
|
|
22
|
+
*/
|
|
23
|
+
private _lastDom;
|
|
24
|
+
/** True if we connected over CDP (don't tear down the browser on close). */
|
|
25
|
+
private _attached;
|
|
26
|
+
constructor();
|
|
27
|
+
launch(opts?: {
|
|
28
|
+
headless?: boolean;
|
|
29
|
+
viewport?: {
|
|
30
|
+
width: number;
|
|
31
|
+
height: number;
|
|
32
|
+
};
|
|
33
|
+
storageState?: string;
|
|
34
|
+
recordVideo?: boolean | {
|
|
35
|
+
dir: string;
|
|
36
|
+
};
|
|
37
|
+
stealth?: boolean | StealthConfig;
|
|
38
|
+
humanize?: boolean | HumanizeConfig;
|
|
39
|
+
cdpUrl?: string;
|
|
40
|
+
}): Promise<void>;
|
|
41
|
+
saveStorageState(path: string): Promise<void>;
|
|
42
|
+
navigate(url: string): Promise<void>;
|
|
43
|
+
back(): Promise<void>;
|
|
44
|
+
screenshot(): Promise<Buffer>;
|
|
45
|
+
/** Viewport size in CSS pixels (matches screenshot dimensions). */
|
|
46
|
+
get viewport(): {
|
|
47
|
+
width: number;
|
|
48
|
+
height: number;
|
|
49
|
+
};
|
|
50
|
+
/** Most recent DOM snapshot. Each entry has a stable `index`. */
|
|
51
|
+
get lastDom(): DomElement[];
|
|
52
|
+
click(x: number, y: number): Promise<void>;
|
|
53
|
+
type(text: string): Promise<void>;
|
|
54
|
+
clickAndType(x: number, y: number, text: string): Promise<void>;
|
|
55
|
+
pressKey(key: string): Promise<void>;
|
|
56
|
+
/**
|
|
57
|
+
* Send arbitrary keyboard keys / shortcuts. Accepts a single
|
|
58
|
+
* Playwright key spec (`"Enter"`, `"Control+l"`, `"Shift+ArrowDown"`)
|
|
59
|
+
* or a space-separated sequence (`"Tab Tab Enter"`).
|
|
60
|
+
*/
|
|
61
|
+
sendKeys(keys: string): Promise<void>;
|
|
62
|
+
scroll(direction: "up" | "down", amount?: number): Promise<void>;
|
|
63
|
+
/**
|
|
64
|
+
* Build a Playwright locator for a DOM-snapshot index. Each `extractDOM`
|
|
65
|
+
* call tags surviving elements with `data-bua-idx="<n>"`; we resolve by
|
|
66
|
+
* that attribute. Returns null if the index is unknown.
|
|
67
|
+
*/
|
|
68
|
+
private locatorForIndex;
|
|
69
|
+
/**
|
|
70
|
+
* Click an element by its DOM-snapshot index. The most reliable click
|
|
71
|
+
* path on dynamic pages — survives layout shifts and DPR oddities.
|
|
72
|
+
*/
|
|
73
|
+
clickByIndex(index: number, opts?: {
|
|
74
|
+
timeout?: number;
|
|
75
|
+
}): Promise<boolean>;
|
|
76
|
+
/**
|
|
77
|
+
* Focus an indexed input, optionally clear it, and type. Returns false
|
|
78
|
+
* if the index couldn't be resolved or the input couldn't be focused.
|
|
79
|
+
*/
|
|
80
|
+
inputByIndex(index: number, text: string, opts?: {
|
|
81
|
+
clear?: boolean;
|
|
82
|
+
submit?: boolean;
|
|
83
|
+
timeout?: number;
|
|
84
|
+
}): Promise<boolean>;
|
|
85
|
+
uploadFileByIndex(index: number, path: string): Promise<boolean>;
|
|
86
|
+
/** Scroll the indexed element into view (no click). */
|
|
87
|
+
scrollIntoViewByIndex(index: number): Promise<boolean>;
|
|
88
|
+
/**
|
|
89
|
+
* Deterministic, DOM-based click using Playwright's text locator.
|
|
90
|
+
*
|
|
91
|
+
* Returns `true` if a matching, visible, clickable element was found and
|
|
92
|
+
* clicked within `timeout` ms; `false` otherwise (so the caller can fall
|
|
93
|
+
* back to coordinate clicking). Substring-matches by default — e.g.
|
|
94
|
+
* `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
|
|
95
|
+
*/
|
|
96
|
+
clickByText(keyword: string, opts?: {
|
|
97
|
+
timeout?: number;
|
|
98
|
+
}): Promise<boolean>;
|
|
99
|
+
/**
|
|
100
|
+
* Scroll the first occurrence of `text` into view. Returns false if no
|
|
101
|
+
* match was found within the timeout.
|
|
102
|
+
*/
|
|
103
|
+
findText(text: string, opts?: {
|
|
104
|
+
timeout?: number;
|
|
105
|
+
}): Promise<boolean>;
|
|
106
|
+
/**
|
|
107
|
+
* Read the options of a native `<select>` at the given DOM-snapshot
|
|
108
|
+
* index. Returns `[]` if the element is not a `<select>`.
|
|
109
|
+
*/
|
|
110
|
+
dropdownOptions(index: number): Promise<{
|
|
111
|
+
value: string;
|
|
112
|
+
label: string;
|
|
113
|
+
selected: boolean;
|
|
114
|
+
}[]>;
|
|
115
|
+
/**
|
|
116
|
+
* Select an option in a native `<select>` by its visible text or value.
|
|
117
|
+
* Returns false if the element isn't a `<select>` or no option matched.
|
|
118
|
+
*/
|
|
119
|
+
selectDropdown(index: number, text: string): Promise<boolean>;
|
|
120
|
+
/**
|
|
121
|
+
* Run arbitrary JS in the page context. The caller is responsible for
|
|
122
|
+
* gating this behind a config flag — the BrowserAgent only routes the
|
|
123
|
+
* `evaluate` action here when `allowEvaluate: true`.
|
|
124
|
+
*
|
|
125
|
+
* The code is wrapped in `(async () => { ... })()` and the return value
|
|
126
|
+
* is coerced to a string for the model.
|
|
127
|
+
*/
|
|
128
|
+
evaluate(code: string): Promise<string>;
|
|
129
|
+
/**
|
|
130
|
+
* Returns a clean text representation of the visible page body, with
|
|
131
|
+
* optional link extraction. Used by the BrowserAgent's `extract` action
|
|
132
|
+
* — the text is passed to a (usually cheap) LLM with the user's query.
|
|
133
|
+
*/
|
|
134
|
+
pageText(opts?: {
|
|
135
|
+
extractLinks?: boolean;
|
|
136
|
+
maxChars?: number;
|
|
137
|
+
}): Promise<string>;
|
|
138
|
+
/**
|
|
139
|
+
* Snapshot the interactive elements visible in the viewport, tag each
|
|
140
|
+
* with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
|
|
141
|
+
* return:
|
|
142
|
+
* - `text`: a human-readable string fed to the model
|
|
143
|
+
* - `elements`: the structured list with stable indices
|
|
144
|
+
* - `scroll`: spatial context (pages above/below, hidden interactive count)
|
|
145
|
+
*
|
|
146
|
+
* Five properties matter for accuracy:
|
|
147
|
+
* - **Hit-tested**: each listed coordinate / index actually reaches the
|
|
148
|
+
* labeled element (overlays / occlusion skip the entry).
|
|
149
|
+
* - **Visibility filtered (with parent chain)**: an element is dropped
|
|
150
|
+
* if itself OR any ancestor is `display:none`, `visibility:hidden`,
|
|
151
|
+
* `pointer-events:none`, or near-zero opacity.
|
|
152
|
+
* - **Shadow DOM piercing**: traverses open shadow roots so custom
|
|
153
|
+
* elements / web components are visible to the agent.
|
|
154
|
+
* - **Same-origin iframes**: walks into each accessible iframe and
|
|
155
|
+
* includes its interactive elements (offset by the iframe's screen
|
|
156
|
+
* position so the coordinates the model sees are still viewport-
|
|
157
|
+
* relative).
|
|
158
|
+
* - **`cursor: pointer` fallback pass**: catches custom React widgets
|
|
159
|
+
* that have no semantic role/href/onclick but are clickable.
|
|
160
|
+
*/
|
|
161
|
+
extractDOM(opts?: {
|
|
162
|
+
maxElements?: number;
|
|
163
|
+
}): Promise<DomSnapshot>;
|
|
164
|
+
/**
|
|
165
|
+
* Install the `__buaExtract` global on the main page. Idempotent —
|
|
166
|
+
* subsequent calls are no-ops.
|
|
167
|
+
*/
|
|
168
|
+
private installExtractorScript;
|
|
169
|
+
/**
|
|
170
|
+
* The extractor source. Lives in its own method so we can also inject
|
|
171
|
+
* it into iframes that haven't yet had it loaded.
|
|
172
|
+
*
|
|
173
|
+
* This function intentionally runs entirely in the page context. It:
|
|
174
|
+
* - traverses the regular DOM + open shadow roots (deep)
|
|
175
|
+
* - applies a parent-chain visibility filter
|
|
176
|
+
* - applies a `cursor:pointer` second pass for custom widgets
|
|
177
|
+
* - hit-tests each candidate at its center to avoid overlay collisions
|
|
178
|
+
* - returns scroll context (pages above/below, hidden counts)
|
|
179
|
+
* - tags survivors with `data-bua-idx` for indexed actions
|
|
180
|
+
*/
|
|
181
|
+
private extractorScriptSource;
|
|
182
|
+
getPageInfo(): Promise<PageInfo>;
|
|
183
|
+
waitForStable(minWait?: number): Promise<void>;
|
|
184
|
+
newTab(url?: string): Promise<string>;
|
|
185
|
+
switchTab(tabId: string): Promise<void>;
|
|
186
|
+
closeTab(tabId: string): Promise<void>;
|
|
187
|
+
visibleText(maxChars?: number): Promise<string>;
|
|
188
|
+
/**
|
|
189
|
+
* Grep visible page text. No LLM. Used by `search_page`.
|
|
190
|
+
*/
|
|
191
|
+
searchPage(opts: {
|
|
192
|
+
pattern: string;
|
|
193
|
+
regex?: boolean;
|
|
194
|
+
caseSensitive?: boolean;
|
|
195
|
+
maxResults?: number;
|
|
196
|
+
contextChars?: number;
|
|
197
|
+
}): Promise<Array<{
|
|
198
|
+
match: string;
|
|
199
|
+
context: string;
|
|
200
|
+
index: number;
|
|
201
|
+
}>>;
|
|
202
|
+
/**
|
|
203
|
+
* querySelectorAll over the page. No LLM. Used by `find_elements`.
|
|
204
|
+
*/
|
|
205
|
+
findElements(selector: string, opts?: {
|
|
206
|
+
maxResults?: number;
|
|
207
|
+
}): Promise<Array<{
|
|
208
|
+
tag: string;
|
|
209
|
+
text: string;
|
|
210
|
+
href?: string;
|
|
211
|
+
}>>;
|
|
212
|
+
listTabs(): {
|
|
213
|
+
id: string;
|
|
214
|
+
url: string;
|
|
215
|
+
active: boolean;
|
|
216
|
+
}[];
|
|
217
|
+
get currentTabId(): string;
|
|
218
|
+
getVideoPath(tabId?: string): Promise<string | null>;
|
|
219
|
+
get videoDir(): string | undefined;
|
|
220
|
+
close(): Promise<void>;
|
|
221
|
+
/** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
|
|
222
|
+
private jitter;
|
|
223
|
+
/**
|
|
224
|
+
* Safety net: clamp coordinates returned by the vision model to the actual
|
|
225
|
+
* viewport. If a model occasionally returns image-space coordinates from a
|
|
226
|
+
* 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
|
|
227
|
+
* from clicking at e.g. (2200, 1300) and either erroring or landing on a
|
|
228
|
+
* random off-screen element.
|
|
229
|
+
*/
|
|
230
|
+
private clampToViewport;
|
|
231
|
+
/**
|
|
232
|
+
* Simulate human mouse movement using smoothstep interpolation.
|
|
233
|
+
*/
|
|
234
|
+
private humanMouseMove;
|
|
235
|
+
/** Small random pause after an interaction. */
|
|
236
|
+
private humanPause;
|
|
237
|
+
private randInt;
|
|
238
|
+
private ensurePage;
|
|
239
|
+
private ensureContext;
|
|
240
|
+
private sleep;
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=browser-provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser-provider.d.ts","sourceRoot":"","sources":["../src/browser-provider.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAoB,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAErH;;;;;;GAMG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,SAAS,CAAoC;IACrD,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAC,CAA2B;IAC7C;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAoB;IACpC,4EAA4E;IAC5E,OAAO,CAAC,SAAS,CAAS;;IAQpB,MAAM,CAAC,IAAI,CAAC,EAAE;QAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,QAAQ,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,OAAO,GAAG;YAAE,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,OAAO,CAAC,EAAE,OAAO,GAAG,aAAa,CAAC;QAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,cAAc,CAAC;QACpC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyGX,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7C,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASpC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAOrB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAanC,mEAAmE;IACnE,IAAI,QAAQ,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAEhD;IAED,iEAAiE;IACjE,IAAI,OAAO,IAAI,UAAU,EAAE,CAE1B;IAIK,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAc1C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjC,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/D,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C;;;;OAIG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrC,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBtE;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAMvB;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IAchF;;;OAGG;IACG,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAC7D,OAAO,CAAC,OAAO,CAAC;IA0Bb,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAatE,uDAAuD;IACjD,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAc5D;;;;;;;OAOG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IAajF;;;OAGG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IAc3E;;;OAGG;IACG,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;IAkBpG;;;OAGG;IACG,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqBnE;;;;;;;OAOG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAsB7C;;;;OAIG;IACG,QAAQ,CAAC,IAAI,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IA0CrF;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,UAAU,CAAC,IAAI,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IAyFvE;;;OAGG;YACW,sBAAsB;IAKpC;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,qBAAqB;IA4KvB,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAYhC,aAAa,CAAC,OAAO,SAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3C,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAarC,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQvC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBtC,WAAW,CAAC,QAAQ,SAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAQnD;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAkCrE;;OAEG;IACG,YAAY,CAChB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAC7B,OAAO,CAAC,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAwB/D,QAAQ,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,EAAE;IAQ1D,IAAI,YAAY,IAAI,MAAM,CAEzB;IAIK,YAAY,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAa1D,IAAI,QAAQ,IAAI,MAAM,GAAG,SAAS,CAEjC;IAIK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B5B,kFAAkF;IAClF,OAAO,CAAC,MAAM;IAMd;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAMvB;;OAEG;YACW,cAAc;IAiB5B,+CAA+C;YACjC,UAAU;IAMxB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,KAAK;CAGd"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Secure credential store for BrowserAgent.
|
|
3
|
+
*
|
|
4
|
+
* Secrets are stored in memory and NEVER sent to the LLM.
|
|
5
|
+
* The model works with placeholders (e.g. `{{email}}`, `{{password}}`),
|
|
6
|
+
* and the agent resolves them to real values only at execution time.
|
|
7
|
+
*/
|
|
8
|
+
export declare class CredentialVault {
|
|
9
|
+
private secrets;
|
|
10
|
+
constructor(initial?: Record<string, string>);
|
|
11
|
+
/** Store a credential. Key names become the placeholder: `{{key}}`. */
|
|
12
|
+
set(key: string, value: string): this;
|
|
13
|
+
/** Retrieve a credential value. Returns undefined if not found. */
|
|
14
|
+
get(key: string): string | undefined;
|
|
15
|
+
has(key: string): boolean;
|
|
16
|
+
/** List available placeholder names (never exposes values). */
|
|
17
|
+
keys(): string[];
|
|
18
|
+
/**
|
|
19
|
+
* Load credentials from environment variables.
|
|
20
|
+
* Maps env var names to placeholder keys.
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* vault.fromEnv({ email: "LOGIN_EMAIL", password: "LOGIN_PASS" });
|
|
24
|
+
*/
|
|
25
|
+
fromEnv(mapping: Record<string, string>): this;
|
|
26
|
+
/**
|
|
27
|
+
* Replace `{{key}}` placeholders in text with actual credential values.
|
|
28
|
+
* Used internally by BrowserAgent right before executing a type action.
|
|
29
|
+
*/
|
|
30
|
+
resolve(text: string): string;
|
|
31
|
+
/**
|
|
32
|
+
* Replace any occurrence of real credential values in text with
|
|
33
|
+
* their `{{key}}` placeholder. Used to sanitize logs and action history.
|
|
34
|
+
*/
|
|
35
|
+
mask(text: string): string;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=credential-vault.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"credential-vault.d.ts","sourceRoot":"","sources":["../src/credential-vault.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAA6B;gBAEhC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAQ5C,uEAAuE;IACvE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAKrC,mEAAmE;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIpC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIzB,+DAA+D;IAC/D,IAAI,IAAI,MAAM,EAAE;IAIhB;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAQ9C;;;OAGG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAM7B;;;OAGG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;CAS3B"}
|