@agentium/browser 2.0.4 → 2.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +90 -32
- package/dist/index.d.cts +17 -2
- package/dist/index.d.ts +17 -2
- package/dist/index.js +90 -32
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -206,10 +206,14 @@ var BrowserProvider = class {
|
|
|
206
206
|
const launchOpts = {
|
|
207
207
|
headless: opts?.headless ?? true
|
|
208
208
|
};
|
|
209
|
+
const windowSizeArg = `--window-size=${this._viewport.width},${this._viewport.height}`;
|
|
210
|
+
const windowPositionArg = "--window-position=0,0";
|
|
209
211
|
if (stealthEnabled) {
|
|
210
212
|
const { args, proxy } = buildStealthLaunchArgs(stealthCfg);
|
|
211
|
-
launchOpts.args = args;
|
|
213
|
+
launchOpts.args = [...args, windowSizeArg, windowPositionArg];
|
|
212
214
|
if (proxy) launchOpts.proxy = proxy;
|
|
215
|
+
} else {
|
|
216
|
+
launchOpts.args = [windowSizeArg, windowPositionArg];
|
|
213
217
|
}
|
|
214
218
|
this.browser = await chromium.launch(launchOpts);
|
|
215
219
|
let contextOpts;
|
|
@@ -259,12 +263,21 @@ var BrowserProvider = class {
|
|
|
259
263
|
// ── Screenshot ───────────────────────────────────────────────────────
|
|
260
264
|
async screenshot() {
|
|
261
265
|
this.ensurePage();
|
|
262
|
-
return await this.page.screenshot({
|
|
266
|
+
return await this.page.screenshot({
|
|
267
|
+
type: "png",
|
|
268
|
+
fullPage: false,
|
|
269
|
+
scale: "css"
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
/** Viewport size in CSS pixels (matches screenshot dimensions). */
|
|
273
|
+
get viewport() {
|
|
274
|
+
return this._viewport;
|
|
263
275
|
}
|
|
264
276
|
// ── Interaction (with optional humanize) ─────────────────────────────
|
|
265
277
|
async click(x, y) {
|
|
266
278
|
this.ensurePage();
|
|
267
|
-
const [
|
|
279
|
+
const [cx, cy] = this.clampToViewport(x, y);
|
|
280
|
+
const [fx, fy] = this.jitter(cx, cy);
|
|
268
281
|
if (this._humanize?.mouseMovement) {
|
|
269
282
|
await this.humanMouseMove(fx, fy);
|
|
270
283
|
}
|
|
@@ -278,9 +291,10 @@ var BrowserProvider = class {
|
|
|
278
291
|
await this.humanPause();
|
|
279
292
|
}
|
|
280
293
|
async clickAndType(x, y, text) {
|
|
281
|
-
|
|
294
|
+
const [cx, cy] = this.clampToViewport(x, y);
|
|
295
|
+
await this.click(cx, cy);
|
|
282
296
|
await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
|
|
283
|
-
const [fx, fy] = this.jitter(
|
|
297
|
+
const [fx, fy] = this.jitter(cx, cy);
|
|
284
298
|
await this.page.mouse.click(fx, fy, { clickCount: 3 });
|
|
285
299
|
await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
|
|
286
300
|
await this.type(text);
|
|
@@ -309,8 +323,10 @@ var BrowserProvider = class {
|
|
|
309
323
|
// ── DOM Extraction ───────────────────────────────────────────────────
|
|
310
324
|
async extractDOM(opts) {
|
|
311
325
|
this.ensurePage();
|
|
312
|
-
const max = opts?.maxElements ??
|
|
326
|
+
const max = opts?.maxElements ?? 120;
|
|
313
327
|
const elements = await this.page.evaluate((limit) => {
|
|
328
|
+
const doc = globalThis.document;
|
|
329
|
+
const win = globalThis.window;
|
|
314
330
|
const selectors = [
|
|
315
331
|
"a[href]",
|
|
316
332
|
"button",
|
|
@@ -321,37 +337,62 @@ var BrowserProvider = class {
|
|
|
321
337
|
"[role='link']",
|
|
322
338
|
"[role='tab']",
|
|
323
339
|
"[role='menuitem']",
|
|
340
|
+
"[role='checkbox']",
|
|
341
|
+
"[role='radio']",
|
|
342
|
+
"[role='switch']",
|
|
343
|
+
"[role='combobox']",
|
|
344
|
+
"[role='option']",
|
|
324
345
|
"[onclick]",
|
|
325
|
-
"[contenteditable='true']"
|
|
346
|
+
"[contenteditable='true']",
|
|
347
|
+
"[tabindex]:not([tabindex='-1'])"
|
|
326
348
|
];
|
|
327
|
-
const all =
|
|
328
|
-
const
|
|
329
|
-
|
|
330
|
-
const
|
|
331
|
-
const
|
|
349
|
+
const all = Array.from(doc.querySelectorAll(selectors.join(",")));
|
|
350
|
+
const vh = win.innerHeight;
|
|
351
|
+
const vw = win.innerWidth;
|
|
352
|
+
const entries = [];
|
|
353
|
+
const seen = /* @__PURE__ */ new Set();
|
|
332
354
|
for (const el of all) {
|
|
333
|
-
if (
|
|
355
|
+
if (seen.has(el)) continue;
|
|
334
356
|
const rect = el.getBoundingClientRect();
|
|
335
|
-
if (rect.width
|
|
357
|
+
if (rect.width < 4 || rect.height < 4) continue;
|
|
336
358
|
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
337
359
|
if (rect.right < 0 || rect.left > vw) continue;
|
|
360
|
+
const style = win.getComputedStyle(el);
|
|
361
|
+
if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
|
|
365
|
+
const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
|
|
366
|
+
try {
|
|
367
|
+
const hit = doc.elementFromPoint(cx, cy);
|
|
368
|
+
if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
} catch {
|
|
372
|
+
}
|
|
373
|
+
seen.add(el);
|
|
338
374
|
const tag = el.tagName.toLowerCase();
|
|
339
375
|
const role = el.getAttribute("role") || tag;
|
|
340
376
|
const type = el.getAttribute("type") || "";
|
|
341
|
-
const
|
|
377
|
+
const innerText = el.innerText || el.textContent || "";
|
|
378
|
+
const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
|
|
342
379
|
const placeholder = el.getAttribute("placeholder") || "";
|
|
343
380
|
const ariaLabel = el.getAttribute("aria-label") || "";
|
|
381
|
+
const title = el.getAttribute("title") || "";
|
|
382
|
+
const name = el.getAttribute("name") || "";
|
|
344
383
|
const href = el.getAttribute("href") || "";
|
|
345
384
|
const value = el.value || "";
|
|
346
|
-
|
|
347
|
-
const cy = Math.round(rect.top + rect.height / 2);
|
|
348
|
-
let label = ariaLabel || text || placeholder || value;
|
|
385
|
+
let label = ariaLabel || text || placeholder || title || value || name;
|
|
349
386
|
if (!label && href) label = href.slice(0, 60);
|
|
350
387
|
if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
|
|
351
|
-
|
|
352
|
-
|
|
388
|
+
entries.push({
|
|
389
|
+
cx,
|
|
390
|
+
cy,
|
|
391
|
+
line: `[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`
|
|
392
|
+
});
|
|
353
393
|
}
|
|
354
|
-
|
|
394
|
+
entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
|
|
395
|
+
return entries.slice(0, limit).map((e) => e.line).join("\n");
|
|
355
396
|
}, max);
|
|
356
397
|
return elements;
|
|
357
398
|
}
|
|
@@ -459,6 +500,18 @@ var BrowserProvider = class {
|
|
|
459
500
|
const j = this._humanize.clickJitter;
|
|
460
501
|
return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
|
|
461
502
|
}
|
|
503
|
+
/**
|
|
504
|
+
* Safety net: clamp coordinates returned by the vision model to the actual
|
|
505
|
+
* viewport. If a model occasionally returns image-space coordinates from a
|
|
506
|
+
* 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
|
|
507
|
+
* from clicking at e.g. (2200, 1300) and either erroring or landing on a
|
|
508
|
+
* random off-screen element.
|
|
509
|
+
*/
|
|
510
|
+
clampToViewport(x, y) {
|
|
511
|
+
const cx = Math.max(0, Math.min(this._viewport.width - 1, Math.round(x)));
|
|
512
|
+
const cy = Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)));
|
|
513
|
+
return [cx, cy];
|
|
514
|
+
}
|
|
462
515
|
/**
|
|
463
516
|
* Simulate human mouse movement using Bézier-like interpolation.
|
|
464
517
|
* Moves from the current mouse position to the target in small steps.
|
|
@@ -502,8 +555,12 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
|
|
|
502
555
|
const lines = [
|
|
503
556
|
`You are a browser automation agent. You receive a screenshot of a web browser and decide what action to take next to complete the user's task.`,
|
|
504
557
|
``,
|
|
505
|
-
`## Viewport`,
|
|
506
|
-
|
|
558
|
+
`## Viewport & Coordinate System`,
|
|
559
|
+
`- The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels.`,
|
|
560
|
+
`- The screenshot you are given has EXACTLY the same dimensions: ${viewport.width}\xD7${viewport.height} pixels (top-left = 0,0).`,
|
|
561
|
+
`- ALL coordinates you return MUST be within 0..${viewport.width - 1} for x and 0..${viewport.height - 1} for y.`,
|
|
562
|
+
`- Coordinates outside this range will be rejected \u2014 your click will go nowhere.`,
|
|
563
|
+
`- If a "Interactive elements on page" list is provided below, PREFER its coordinates over your own visual estimation. They are exact element centers in the same coordinate system.`,
|
|
507
564
|
``,
|
|
508
565
|
`## Available Actions`,
|
|
509
566
|
`Respond with a JSON object containing one of these actions:`,
|
|
@@ -544,14 +601,15 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
|
|
|
544
601
|
`## Rules`,
|
|
545
602
|
`1. ALWAYS look at the screenshot carefully before deciding your action.`,
|
|
546
603
|
`2. Provide coordinates that target the CENTER of the element you want to interact with.`,
|
|
547
|
-
`3.
|
|
548
|
-
`4.
|
|
549
|
-
`5.
|
|
550
|
-
`6. If
|
|
551
|
-
`7. If
|
|
552
|
-
`8.
|
|
553
|
-
`9.
|
|
554
|
-
`10.
|
|
604
|
+
`3. When an "Interactive elements on page" list is provided, identify the target element in that list by its visible label (button text, aria-label, placeholder, link href) and use ITS coordinates \u2014 they are the ground truth. Only fall back to visual estimation when no matching element is listed.`,
|
|
605
|
+
`4. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
|
|
606
|
+
`5. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
|
|
607
|
+
`6. If your previous click landed on the wrong element (e.g. page didn't change or a different element became focused), the target was probably misaligned \u2014 re-read the elements list and pick a different coordinate; do NOT repeat the same click.`,
|
|
608
|
+
`7. If a page is loading or blank, use "wait" with a short delay and try again.`,
|
|
609
|
+
`8. If you see a cookie banner, consent dialog, or popup, dismiss it FIRST before proceeding with the task \u2014 these often intercept clicks on elements behind them.`,
|
|
610
|
+
`9. NEVER hallucinate content. Only report what you can actually see on the screen.`,
|
|
611
|
+
`10. When the task is fully complete, use "done" immediately with a comprehensive result.`,
|
|
612
|
+
`11. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
|
|
555
613
|
``,
|
|
556
614
|
`## Response Format`,
|
|
557
615
|
`Respond with ONLY a valid JSON object. No markdown, no explanation, just the JSON action.`
|
|
@@ -657,7 +715,7 @@ var BrowserAgent = class {
|
|
|
657
715
|
this.defaultStartUrl = config.startUrl;
|
|
658
716
|
this.waitAfterAction = config.waitAfterAction ?? 1500;
|
|
659
717
|
this.maxRepeats = config.maxRepeats ?? 3;
|
|
660
|
-
this.useDOM = config.useDOM ??
|
|
718
|
+
this.useDOM = config.useDOM ?? true;
|
|
661
719
|
this.storageState = config.storageState;
|
|
662
720
|
this.recordVideo = config.recordVideo;
|
|
663
721
|
this.credentials = config.credentials;
|
package/dist/index.d.cts
CHANGED
|
@@ -91,8 +91,10 @@ interface BrowserAgentConfig {
|
|
|
91
91
|
/** Max consecutive identical actions before the agent auto-fails. Default: 3 */
|
|
92
92
|
maxRepeats?: number;
|
|
93
93
|
/**
|
|
94
|
-
* Include a simplified DOM/accessibility tree
|
|
95
|
-
*
|
|
94
|
+
* Include a simplified DOM/accessibility tree (each interactive element
|
|
95
|
+
* tagged with its exact center coordinates) alongside the screenshot.
|
|
96
|
+
* Dramatically improves click accuracy — strongly recommended.
|
|
97
|
+
* Default: true
|
|
96
98
|
*/
|
|
97
99
|
useDOM?: boolean;
|
|
98
100
|
/**
|
|
@@ -290,6 +292,11 @@ declare class BrowserProvider {
|
|
|
290
292
|
navigate(url: string): Promise<void>;
|
|
291
293
|
back(): Promise<void>;
|
|
292
294
|
screenshot(): Promise<Buffer>;
|
|
295
|
+
/** Viewport size in CSS pixels (matches screenshot dimensions). */
|
|
296
|
+
get viewport(): {
|
|
297
|
+
width: number;
|
|
298
|
+
height: number;
|
|
299
|
+
};
|
|
293
300
|
click(x: number, y: number): Promise<void>;
|
|
294
301
|
type(text: string): Promise<void>;
|
|
295
302
|
clickAndType(x: number, y: number, text: string): Promise<void>;
|
|
@@ -314,6 +321,14 @@ declare class BrowserProvider {
|
|
|
314
321
|
close(): Promise<void>;
|
|
315
322
|
/** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
|
|
316
323
|
private jitter;
|
|
324
|
+
/**
|
|
325
|
+
* Safety net: clamp coordinates returned by the vision model to the actual
|
|
326
|
+
* viewport. If a model occasionally returns image-space coordinates from a
|
|
327
|
+
* 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
|
|
328
|
+
* from clicking at e.g. (2200, 1300) and either erroring or landing on a
|
|
329
|
+
* random off-screen element.
|
|
330
|
+
*/
|
|
331
|
+
private clampToViewport;
|
|
317
332
|
/**
|
|
318
333
|
* Simulate human mouse movement using Bézier-like interpolation.
|
|
319
334
|
* Moves from the current mouse position to the target in small steps.
|
package/dist/index.d.ts
CHANGED
|
@@ -91,8 +91,10 @@ interface BrowserAgentConfig {
|
|
|
91
91
|
/** Max consecutive identical actions before the agent auto-fails. Default: 3 */
|
|
92
92
|
maxRepeats?: number;
|
|
93
93
|
/**
|
|
94
|
-
* Include a simplified DOM/accessibility tree
|
|
95
|
-
*
|
|
94
|
+
* Include a simplified DOM/accessibility tree (each interactive element
|
|
95
|
+
* tagged with its exact center coordinates) alongside the screenshot.
|
|
96
|
+
* Dramatically improves click accuracy — strongly recommended.
|
|
97
|
+
* Default: true
|
|
96
98
|
*/
|
|
97
99
|
useDOM?: boolean;
|
|
98
100
|
/**
|
|
@@ -290,6 +292,11 @@ declare class BrowserProvider {
|
|
|
290
292
|
navigate(url: string): Promise<void>;
|
|
291
293
|
back(): Promise<void>;
|
|
292
294
|
screenshot(): Promise<Buffer>;
|
|
295
|
+
/** Viewport size in CSS pixels (matches screenshot dimensions). */
|
|
296
|
+
get viewport(): {
|
|
297
|
+
width: number;
|
|
298
|
+
height: number;
|
|
299
|
+
};
|
|
293
300
|
click(x: number, y: number): Promise<void>;
|
|
294
301
|
type(text: string): Promise<void>;
|
|
295
302
|
clickAndType(x: number, y: number, text: string): Promise<void>;
|
|
@@ -314,6 +321,14 @@ declare class BrowserProvider {
|
|
|
314
321
|
close(): Promise<void>;
|
|
315
322
|
/** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
|
|
316
323
|
private jitter;
|
|
324
|
+
/**
|
|
325
|
+
* Safety net: clamp coordinates returned by the vision model to the actual
|
|
326
|
+
* viewport. If a model occasionally returns image-space coordinates from a
|
|
327
|
+
* 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
|
|
328
|
+
* from clicking at e.g. (2200, 1300) and either erroring or landing on a
|
|
329
|
+
* random off-screen element.
|
|
330
|
+
*/
|
|
331
|
+
private clampToViewport;
|
|
317
332
|
/**
|
|
318
333
|
* Simulate human mouse movement using Bézier-like interpolation.
|
|
319
334
|
* Moves from the current mouse position to the target in small steps.
|
package/dist/index.js
CHANGED
|
@@ -168,10 +168,14 @@ var BrowserProvider = class {
|
|
|
168
168
|
const launchOpts = {
|
|
169
169
|
headless: opts?.headless ?? true
|
|
170
170
|
};
|
|
171
|
+
const windowSizeArg = `--window-size=${this._viewport.width},${this._viewport.height}`;
|
|
172
|
+
const windowPositionArg = "--window-position=0,0";
|
|
171
173
|
if (stealthEnabled) {
|
|
172
174
|
const { args, proxy } = buildStealthLaunchArgs(stealthCfg);
|
|
173
|
-
launchOpts.args = args;
|
|
175
|
+
launchOpts.args = [...args, windowSizeArg, windowPositionArg];
|
|
174
176
|
if (proxy) launchOpts.proxy = proxy;
|
|
177
|
+
} else {
|
|
178
|
+
launchOpts.args = [windowSizeArg, windowPositionArg];
|
|
175
179
|
}
|
|
176
180
|
this.browser = await chromium.launch(launchOpts);
|
|
177
181
|
let contextOpts;
|
|
@@ -221,12 +225,21 @@ var BrowserProvider = class {
|
|
|
221
225
|
// ── Screenshot ───────────────────────────────────────────────────────
|
|
222
226
|
async screenshot() {
|
|
223
227
|
this.ensurePage();
|
|
224
|
-
return await this.page.screenshot({
|
|
228
|
+
return await this.page.screenshot({
|
|
229
|
+
type: "png",
|
|
230
|
+
fullPage: false,
|
|
231
|
+
scale: "css"
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
/** Viewport size in CSS pixels (matches screenshot dimensions). */
|
|
235
|
+
get viewport() {
|
|
236
|
+
return this._viewport;
|
|
225
237
|
}
|
|
226
238
|
// ── Interaction (with optional humanize) ─────────────────────────────
|
|
227
239
|
async click(x, y) {
|
|
228
240
|
this.ensurePage();
|
|
229
|
-
const [
|
|
241
|
+
const [cx, cy] = this.clampToViewport(x, y);
|
|
242
|
+
const [fx, fy] = this.jitter(cx, cy);
|
|
230
243
|
if (this._humanize?.mouseMovement) {
|
|
231
244
|
await this.humanMouseMove(fx, fy);
|
|
232
245
|
}
|
|
@@ -240,9 +253,10 @@ var BrowserProvider = class {
|
|
|
240
253
|
await this.humanPause();
|
|
241
254
|
}
|
|
242
255
|
async clickAndType(x, y, text) {
|
|
243
|
-
|
|
256
|
+
const [cx, cy] = this.clampToViewport(x, y);
|
|
257
|
+
await this.click(cx, cy);
|
|
244
258
|
await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
|
|
245
|
-
const [fx, fy] = this.jitter(
|
|
259
|
+
const [fx, fy] = this.jitter(cx, cy);
|
|
246
260
|
await this.page.mouse.click(fx, fy, { clickCount: 3 });
|
|
247
261
|
await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
|
|
248
262
|
await this.type(text);
|
|
@@ -271,8 +285,10 @@ var BrowserProvider = class {
|
|
|
271
285
|
// ── DOM Extraction ───────────────────────────────────────────────────
|
|
272
286
|
async extractDOM(opts) {
|
|
273
287
|
this.ensurePage();
|
|
274
|
-
const max = opts?.maxElements ??
|
|
288
|
+
const max = opts?.maxElements ?? 120;
|
|
275
289
|
const elements = await this.page.evaluate((limit) => {
|
|
290
|
+
const doc = globalThis.document;
|
|
291
|
+
const win = globalThis.window;
|
|
276
292
|
const selectors = [
|
|
277
293
|
"a[href]",
|
|
278
294
|
"button",
|
|
@@ -283,37 +299,62 @@ var BrowserProvider = class {
|
|
|
283
299
|
"[role='link']",
|
|
284
300
|
"[role='tab']",
|
|
285
301
|
"[role='menuitem']",
|
|
302
|
+
"[role='checkbox']",
|
|
303
|
+
"[role='radio']",
|
|
304
|
+
"[role='switch']",
|
|
305
|
+
"[role='combobox']",
|
|
306
|
+
"[role='option']",
|
|
286
307
|
"[onclick]",
|
|
287
|
-
"[contenteditable='true']"
|
|
308
|
+
"[contenteditable='true']",
|
|
309
|
+
"[tabindex]:not([tabindex='-1'])"
|
|
288
310
|
];
|
|
289
|
-
const all =
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
const
|
|
293
|
-
const
|
|
311
|
+
const all = Array.from(doc.querySelectorAll(selectors.join(",")));
|
|
312
|
+
const vh = win.innerHeight;
|
|
313
|
+
const vw = win.innerWidth;
|
|
314
|
+
const entries = [];
|
|
315
|
+
const seen = /* @__PURE__ */ new Set();
|
|
294
316
|
for (const el of all) {
|
|
295
|
-
if (
|
|
317
|
+
if (seen.has(el)) continue;
|
|
296
318
|
const rect = el.getBoundingClientRect();
|
|
297
|
-
if (rect.width
|
|
319
|
+
if (rect.width < 4 || rect.height < 4) continue;
|
|
298
320
|
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
299
321
|
if (rect.right < 0 || rect.left > vw) continue;
|
|
322
|
+
const style = win.getComputedStyle(el);
|
|
323
|
+
if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
|
|
327
|
+
const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
|
|
328
|
+
try {
|
|
329
|
+
const hit = doc.elementFromPoint(cx, cy);
|
|
330
|
+
if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
} catch {
|
|
334
|
+
}
|
|
335
|
+
seen.add(el);
|
|
300
336
|
const tag = el.tagName.toLowerCase();
|
|
301
337
|
const role = el.getAttribute("role") || tag;
|
|
302
338
|
const type = el.getAttribute("type") || "";
|
|
303
|
-
const
|
|
339
|
+
const innerText = el.innerText || el.textContent || "";
|
|
340
|
+
const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
|
|
304
341
|
const placeholder = el.getAttribute("placeholder") || "";
|
|
305
342
|
const ariaLabel = el.getAttribute("aria-label") || "";
|
|
343
|
+
const title = el.getAttribute("title") || "";
|
|
344
|
+
const name = el.getAttribute("name") || "";
|
|
306
345
|
const href = el.getAttribute("href") || "";
|
|
307
346
|
const value = el.value || "";
|
|
308
|
-
|
|
309
|
-
const cy = Math.round(rect.top + rect.height / 2);
|
|
310
|
-
let label = ariaLabel || text || placeholder || value;
|
|
347
|
+
let label = ariaLabel || text || placeholder || title || value || name;
|
|
311
348
|
if (!label && href) label = href.slice(0, 60);
|
|
312
349
|
if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
|
|
313
|
-
|
|
314
|
-
|
|
350
|
+
entries.push({
|
|
351
|
+
cx,
|
|
352
|
+
cy,
|
|
353
|
+
line: `[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`
|
|
354
|
+
});
|
|
315
355
|
}
|
|
316
|
-
|
|
356
|
+
entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
|
|
357
|
+
return entries.slice(0, limit).map((e) => e.line).join("\n");
|
|
317
358
|
}, max);
|
|
318
359
|
return elements;
|
|
319
360
|
}
|
|
@@ -421,6 +462,18 @@ var BrowserProvider = class {
|
|
|
421
462
|
const j = this._humanize.clickJitter;
|
|
422
463
|
return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
|
|
423
464
|
}
|
|
465
|
+
/**
|
|
466
|
+
* Safety net: clamp coordinates returned by the vision model to the actual
|
|
467
|
+
* viewport. If a model occasionally returns image-space coordinates from a
|
|
468
|
+
* 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
|
|
469
|
+
* from clicking at e.g. (2200, 1300) and either erroring or landing on a
|
|
470
|
+
* random off-screen element.
|
|
471
|
+
*/
|
|
472
|
+
clampToViewport(x, y) {
|
|
473
|
+
const cx = Math.max(0, Math.min(this._viewport.width - 1, Math.round(x)));
|
|
474
|
+
const cy = Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)));
|
|
475
|
+
return [cx, cy];
|
|
476
|
+
}
|
|
424
477
|
/**
|
|
425
478
|
* Simulate human mouse movement using Bézier-like interpolation.
|
|
426
479
|
* Moves from the current mouse position to the target in small steps.
|
|
@@ -464,8 +517,12 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
|
|
|
464
517
|
const lines = [
|
|
465
518
|
`You are a browser automation agent. You receive a screenshot of a web browser and decide what action to take next to complete the user's task.`,
|
|
466
519
|
``,
|
|
467
|
-
`## Viewport`,
|
|
468
|
-
|
|
520
|
+
`## Viewport & Coordinate System`,
|
|
521
|
+
`- The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels.`,
|
|
522
|
+
`- The screenshot you are given has EXACTLY the same dimensions: ${viewport.width}\xD7${viewport.height} pixels (top-left = 0,0).`,
|
|
523
|
+
`- ALL coordinates you return MUST be within 0..${viewport.width - 1} for x and 0..${viewport.height - 1} for y.`,
|
|
524
|
+
`- Coordinates outside this range will be rejected \u2014 your click will go nowhere.`,
|
|
525
|
+
`- If a "Interactive elements on page" list is provided below, PREFER its coordinates over your own visual estimation. They are exact element centers in the same coordinate system.`,
|
|
469
526
|
``,
|
|
470
527
|
`## Available Actions`,
|
|
471
528
|
`Respond with a JSON object containing one of these actions:`,
|
|
@@ -506,14 +563,15 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
|
|
|
506
563
|
`## Rules`,
|
|
507
564
|
`1. ALWAYS look at the screenshot carefully before deciding your action.`,
|
|
508
565
|
`2. Provide coordinates that target the CENTER of the element you want to interact with.`,
|
|
509
|
-
`3.
|
|
510
|
-
`4.
|
|
511
|
-
`5.
|
|
512
|
-
`6. If
|
|
513
|
-
`7. If
|
|
514
|
-
`8.
|
|
515
|
-
`9.
|
|
516
|
-
`10.
|
|
566
|
+
`3. When an "Interactive elements on page" list is provided, identify the target element in that list by its visible label (button text, aria-label, placeholder, link href) and use ITS coordinates \u2014 they are the ground truth. Only fall back to visual estimation when no matching element is listed.`,
|
|
567
|
+
`4. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
|
|
568
|
+
`5. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
|
|
569
|
+
`6. If your previous click landed on the wrong element (e.g. page didn't change or a different element became focused), the target was probably misaligned \u2014 re-read the elements list and pick a different coordinate; do NOT repeat the same click.`,
|
|
570
|
+
`7. If a page is loading or blank, use "wait" with a short delay and try again.`,
|
|
571
|
+
`8. If you see a cookie banner, consent dialog, or popup, dismiss it FIRST before proceeding with the task \u2014 these often intercept clicks on elements behind them.`,
|
|
572
|
+
`9. NEVER hallucinate content. Only report what you can actually see on the screen.`,
|
|
573
|
+
`10. When the task is fully complete, use "done" immediately with a comprehensive result.`,
|
|
574
|
+
`11. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
|
|
517
575
|
``,
|
|
518
576
|
`## Response Format`,
|
|
519
577
|
`Respond with ONLY a valid JSON object. No markdown, no explanation, just the JSON action.`
|
|
@@ -619,7 +677,7 @@ var BrowserAgent = class {
|
|
|
619
677
|
this.defaultStartUrl = config.startUrl;
|
|
620
678
|
this.waitAfterAction = config.waitAfterAction ?? 1500;
|
|
621
679
|
this.maxRepeats = config.maxRepeats ?? 3;
|
|
622
|
-
this.useDOM = config.useDOM ??
|
|
680
|
+
this.useDOM = config.useDOM ?? true;
|
|
623
681
|
this.storageState = config.storageState;
|
|
624
682
|
this.recordVideo = config.recordVideo;
|
|
625
683
|
this.credentials = config.credentials;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentium/browser",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.6",
|
|
4
4
|
"description": "Browser automation agent for Agentium using Playwright",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"typescript": "^5.6.0"
|
|
47
47
|
},
|
|
48
48
|
"peerDependencies": {
|
|
49
|
-
"@agentium/core": "^2.0.
|
|
49
|
+
"@agentium/core": "^2.0.6",
|
|
50
50
|
"playwright": ">=1.40.0"
|
|
51
51
|
},
|
|
52
52
|
"peerDependenciesMeta": {
|