@agentium/browser 2.0.4 → 2.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -206,10 +206,14 @@ var BrowserProvider = class {
206
206
  const launchOpts = {
207
207
  headless: opts?.headless ?? true
208
208
  };
209
+ const windowSizeArg = `--window-size=${this._viewport.width},${this._viewport.height}`;
210
+ const windowPositionArg = "--window-position=0,0";
209
211
  if (stealthEnabled) {
210
212
  const { args, proxy } = buildStealthLaunchArgs(stealthCfg);
211
- launchOpts.args = args;
213
+ launchOpts.args = [...args, windowSizeArg, windowPositionArg];
212
214
  if (proxy) launchOpts.proxy = proxy;
215
+ } else {
216
+ launchOpts.args = [windowSizeArg, windowPositionArg];
213
217
  }
214
218
  this.browser = await chromium.launch(launchOpts);
215
219
  let contextOpts;
@@ -259,12 +263,21 @@ var BrowserProvider = class {
259
263
  // ── Screenshot ───────────────────────────────────────────────────────
260
264
  async screenshot() {
261
265
  this.ensurePage();
262
- return await this.page.screenshot({ type: "png", fullPage: false });
266
+ return await this.page.screenshot({
267
+ type: "png",
268
+ fullPage: false,
269
+ scale: "css"
270
+ });
271
+ }
272
+ /** Viewport size in CSS pixels (matches screenshot dimensions). */
273
+ get viewport() {
274
+ return this._viewport;
263
275
  }
264
276
  // ── Interaction (with optional humanize) ─────────────────────────────
265
277
  async click(x, y) {
266
278
  this.ensurePage();
267
- const [fx, fy] = this.jitter(x, y);
279
+ const [cx, cy] = this.clampToViewport(x, y);
280
+ const [fx, fy] = this.jitter(cx, cy);
268
281
  if (this._humanize?.mouseMovement) {
269
282
  await this.humanMouseMove(fx, fy);
270
283
  }
@@ -278,9 +291,10 @@ var BrowserProvider = class {
278
291
  await this.humanPause();
279
292
  }
280
293
  async clickAndType(x, y, text) {
281
- await this.click(x, y);
294
+ const [cx, cy] = this.clampToViewport(x, y);
295
+ await this.click(cx, cy);
282
296
  await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
283
- const [fx, fy] = this.jitter(x, y);
297
+ const [fx, fy] = this.jitter(cx, cy);
284
298
  await this.page.mouse.click(fx, fy, { clickCount: 3 });
285
299
  await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
286
300
  await this.type(text);
@@ -309,8 +323,10 @@ var BrowserProvider = class {
309
323
  // ── DOM Extraction ───────────────────────────────────────────────────
310
324
  async extractDOM(opts) {
311
325
  this.ensurePage();
312
- const max = opts?.maxElements ?? 80;
326
+ const max = opts?.maxElements ?? 120;
313
327
  const elements = await this.page.evaluate((limit) => {
328
+ const doc = globalThis.document;
329
+ const win = globalThis.window;
314
330
  const selectors = [
315
331
  "a[href]",
316
332
  "button",
@@ -321,37 +337,62 @@ var BrowserProvider = class {
321
337
  "[role='link']",
322
338
  "[role='tab']",
323
339
  "[role='menuitem']",
340
+ "[role='checkbox']",
341
+ "[role='radio']",
342
+ "[role='switch']",
343
+ "[role='combobox']",
344
+ "[role='option']",
324
345
  "[onclick]",
325
- "[contenteditable='true']"
346
+ "[contenteditable='true']",
347
+ "[tabindex]:not([tabindex='-1'])"
326
348
  ];
327
- const all = globalThis.document.querySelectorAll(selectors.join(","));
328
- const lines = [];
329
- let count = 0;
330
- const vh = globalThis.window.innerHeight;
331
- const vw = globalThis.window.innerWidth;
349
+ const all = Array.from(doc.querySelectorAll(selectors.join(",")));
350
+ const vh = win.innerHeight;
351
+ const vw = win.innerWidth;
352
+ const entries = [];
353
+ const seen = /* @__PURE__ */ new Set();
332
354
  for (const el of all) {
333
- if (count >= limit) break;
355
+ if (seen.has(el)) continue;
334
356
  const rect = el.getBoundingClientRect();
335
- if (rect.width === 0 || rect.height === 0) continue;
357
+ if (rect.width < 4 || rect.height < 4) continue;
336
358
  if (rect.bottom < 0 || rect.top > vh) continue;
337
359
  if (rect.right < 0 || rect.left > vw) continue;
360
+ const style = win.getComputedStyle(el);
361
+ if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
362
+ continue;
363
+ }
364
+ const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
365
+ const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
366
+ try {
367
+ const hit = doc.elementFromPoint(cx, cy);
368
+ if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
369
+ continue;
370
+ }
371
+ } catch {
372
+ }
373
+ seen.add(el);
338
374
  const tag = el.tagName.toLowerCase();
339
375
  const role = el.getAttribute("role") || tag;
340
376
  const type = el.getAttribute("type") || "";
341
- const text = (el.textContent || "").trim().slice(0, 80);
377
+ const innerText = el.innerText || el.textContent || "";
378
+ const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
342
379
  const placeholder = el.getAttribute("placeholder") || "";
343
380
  const ariaLabel = el.getAttribute("aria-label") || "";
381
+ const title = el.getAttribute("title") || "";
382
+ const name = el.getAttribute("name") || "";
344
383
  const href = el.getAttribute("href") || "";
345
384
  const value = el.value || "";
346
- const cx = Math.round(rect.left + rect.width / 2);
347
- const cy = Math.round(rect.top + rect.height / 2);
348
- let label = ariaLabel || text || placeholder || value;
385
+ let label = ariaLabel || text || placeholder || title || value || name;
349
386
  if (!label && href) label = href.slice(0, 60);
350
387
  if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
351
- lines.push(`[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`);
352
- count++;
388
+ entries.push({
389
+ cx,
390
+ cy,
391
+ line: `[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`
392
+ });
353
393
  }
354
- return lines.join("\n");
394
+ entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
395
+ return entries.slice(0, limit).map((e) => e.line).join("\n");
355
396
  }, max);
356
397
  return elements;
357
398
  }
@@ -459,6 +500,18 @@ var BrowserProvider = class {
459
500
  const j = this._humanize.clickJitter;
460
501
  return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
461
502
  }
503
+ /**
504
+ * Safety net: clamp coordinates returned by the vision model to the actual
505
+ * viewport. If a model occasionally returns image-space coordinates from a
506
+ * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
507
+ * from clicking at e.g. (2200, 1300) and either erroring or landing on a
508
+ * random off-screen element.
509
+ */
510
+ clampToViewport(x, y) {
511
+ const cx = Math.max(0, Math.min(this._viewport.width - 1, Math.round(x)));
512
+ const cy = Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)));
513
+ return [cx, cy];
514
+ }
462
515
  /**
463
516
  * Simulate human mouse movement using Bézier-like interpolation.
464
517
  * Moves from the current mouse position to the target in small steps.
@@ -502,8 +555,12 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
502
555
  const lines = [
503
556
  `You are a browser automation agent. You receive a screenshot of a web browser and decide what action to take next to complete the user's task.`,
504
557
  ``,
505
- `## Viewport`,
506
- `The browser viewport is ${viewport.width}\xD7${viewport.height} pixels. All coordinates you provide must be within this range.`,
558
+ `## Viewport & Coordinate System`,
559
+ `- The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels.`,
560
+ `- The screenshot you are given has EXACTLY the same dimensions: ${viewport.width}\xD7${viewport.height} pixels (top-left = 0,0).`,
561
+ `- ALL coordinates you return MUST be within 0..${viewport.width - 1} for x and 0..${viewport.height - 1} for y.`,
562
+ `- Coordinates outside this range will be rejected \u2014 your click will go nowhere.`,
563
+ `- If a "Interactive elements on page" list is provided below, PREFER its coordinates over your own visual estimation. They are exact element centers in the same coordinate system.`,
507
564
  ``,
508
565
  `## Available Actions`,
509
566
  `Respond with a JSON object containing one of these actions:`,
@@ -544,14 +601,15 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
544
601
  `## Rules`,
545
602
  `1. ALWAYS look at the screenshot carefully before deciding your action.`,
546
603
  `2. Provide coordinates that target the CENTER of the element you want to interact with.`,
547
- `3. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
548
- `4. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
549
- `5. If your previous action didn't produce the expected result, try a different approach.`,
550
- `6. If a page is loading or blank, use "wait" with a short delay and try again.`,
551
- `7. If you see a cookie banner or popup, dismiss it first before proceeding with the task.`,
552
- `8. NEVER hallucinate content. Only report what you can actually see on the screen.`,
553
- `9. When the task is fully complete, use "done" immediately with a comprehensive result.`,
554
- `10. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
604
+ `3. When an "Interactive elements on page" list is provided, identify the target element in that list by its visible label (button text, aria-label, placeholder, link href) and use ITS coordinates \u2014 they are the ground truth. Only fall back to visual estimation when no matching element is listed.`,
605
+ `4. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
606
+ `5. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
607
+ `6. If your previous click landed on the wrong element (e.g. page didn't change or a different element became focused), the target was probably misaligned \u2014 re-read the elements list and pick a different coordinate; do NOT repeat the same click.`,
608
+ `7. If a page is loading or blank, use "wait" with a short delay and try again.`,
609
+ `8. If you see a cookie banner, consent dialog, or popup, dismiss it FIRST before proceeding with the task \u2014 these often intercept clicks on elements behind them.`,
610
+ `9. NEVER hallucinate content. Only report what you can actually see on the screen.`,
611
+ `10. When the task is fully complete, use "done" immediately with a comprehensive result.`,
612
+ `11. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
555
613
  ``,
556
614
  `## Response Format`,
557
615
  `Respond with ONLY a valid JSON object. No markdown, no explanation, just the JSON action.`
@@ -657,7 +715,7 @@ var BrowserAgent = class {
657
715
  this.defaultStartUrl = config.startUrl;
658
716
  this.waitAfterAction = config.waitAfterAction ?? 1500;
659
717
  this.maxRepeats = config.maxRepeats ?? 3;
660
- this.useDOM = config.useDOM ?? false;
718
+ this.useDOM = config.useDOM ?? true;
661
719
  this.storageState = config.storageState;
662
720
  this.recordVideo = config.recordVideo;
663
721
  this.credentials = config.credentials;
package/dist/index.d.cts CHANGED
@@ -91,8 +91,10 @@ interface BrowserAgentConfig {
91
91
  /** Max consecutive identical actions before the agent auto-fails. Default: 3 */
92
92
  maxRepeats?: number;
93
93
  /**
94
- * Include a simplified DOM/accessibility tree alongside the screenshot.
95
- * Helps the model target elements more accurately. Default: false
94
+ * Include a simplified DOM/accessibility tree (each interactive element
95
+ * tagged with its exact center coordinates) alongside the screenshot.
96
+ * Dramatically improves click accuracy — strongly recommended.
97
+ * Default: true
96
98
  */
97
99
  useDOM?: boolean;
98
100
  /**
@@ -290,6 +292,11 @@ declare class BrowserProvider {
290
292
  navigate(url: string): Promise<void>;
291
293
  back(): Promise<void>;
292
294
  screenshot(): Promise<Buffer>;
295
+ /** Viewport size in CSS pixels (matches screenshot dimensions). */
296
+ get viewport(): {
297
+ width: number;
298
+ height: number;
299
+ };
293
300
  click(x: number, y: number): Promise<void>;
294
301
  type(text: string): Promise<void>;
295
302
  clickAndType(x: number, y: number, text: string): Promise<void>;
@@ -314,6 +321,14 @@ declare class BrowserProvider {
314
321
  close(): Promise<void>;
315
322
  /** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
316
323
  private jitter;
324
+ /**
325
+ * Safety net: clamp coordinates returned by the vision model to the actual
326
+ * viewport. If a model occasionally returns image-space coordinates from a
327
+ * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
328
+ * from clicking at e.g. (2200, 1300) and either erroring or landing on a
329
+ * random off-screen element.
330
+ */
331
+ private clampToViewport;
317
332
  /**
318
333
  * Simulate human mouse movement using Bézier-like interpolation.
319
334
  * Moves from the current mouse position to the target in small steps.
package/dist/index.d.ts CHANGED
@@ -91,8 +91,10 @@ interface BrowserAgentConfig {
91
91
  /** Max consecutive identical actions before the agent auto-fails. Default: 3 */
92
92
  maxRepeats?: number;
93
93
  /**
94
- * Include a simplified DOM/accessibility tree alongside the screenshot.
95
- * Helps the model target elements more accurately. Default: false
94
+ * Include a simplified DOM/accessibility tree (each interactive element
95
+ * tagged with its exact center coordinates) alongside the screenshot.
96
+ * Dramatically improves click accuracy — strongly recommended.
97
+ * Default: true
96
98
  */
97
99
  useDOM?: boolean;
98
100
  /**
@@ -290,6 +292,11 @@ declare class BrowserProvider {
290
292
  navigate(url: string): Promise<void>;
291
293
  back(): Promise<void>;
292
294
  screenshot(): Promise<Buffer>;
295
+ /** Viewport size in CSS pixels (matches screenshot dimensions). */
296
+ get viewport(): {
297
+ width: number;
298
+ height: number;
299
+ };
293
300
  click(x: number, y: number): Promise<void>;
294
301
  type(text: string): Promise<void>;
295
302
  clickAndType(x: number, y: number, text: string): Promise<void>;
@@ -314,6 +321,14 @@ declare class BrowserProvider {
314
321
  close(): Promise<void>;
315
322
  /** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
316
323
  private jitter;
324
+ /**
325
+ * Safety net: clamp coordinates returned by the vision model to the actual
326
+ * viewport. If a model occasionally returns image-space coordinates from a
327
+ * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
328
+ * from clicking at e.g. (2200, 1300) and either erroring or landing on a
329
+ * random off-screen element.
330
+ */
331
+ private clampToViewport;
317
332
  /**
318
333
  * Simulate human mouse movement using Bézier-like interpolation.
319
334
  * Moves from the current mouse position to the target in small steps.
package/dist/index.js CHANGED
@@ -168,10 +168,14 @@ var BrowserProvider = class {
168
168
  const launchOpts = {
169
169
  headless: opts?.headless ?? true
170
170
  };
171
+ const windowSizeArg = `--window-size=${this._viewport.width},${this._viewport.height}`;
172
+ const windowPositionArg = "--window-position=0,0";
171
173
  if (stealthEnabled) {
172
174
  const { args, proxy } = buildStealthLaunchArgs(stealthCfg);
173
- launchOpts.args = args;
175
+ launchOpts.args = [...args, windowSizeArg, windowPositionArg];
174
176
  if (proxy) launchOpts.proxy = proxy;
177
+ } else {
178
+ launchOpts.args = [windowSizeArg, windowPositionArg];
175
179
  }
176
180
  this.browser = await chromium.launch(launchOpts);
177
181
  let contextOpts;
@@ -221,12 +225,21 @@ var BrowserProvider = class {
221
225
  // ── Screenshot ───────────────────────────────────────────────────────
222
226
  async screenshot() {
223
227
  this.ensurePage();
224
- return await this.page.screenshot({ type: "png", fullPage: false });
228
+ return await this.page.screenshot({
229
+ type: "png",
230
+ fullPage: false,
231
+ scale: "css"
232
+ });
233
+ }
234
+ /** Viewport size in CSS pixels (matches screenshot dimensions). */
235
+ get viewport() {
236
+ return this._viewport;
225
237
  }
226
238
  // ── Interaction (with optional humanize) ─────────────────────────────
227
239
  async click(x, y) {
228
240
  this.ensurePage();
229
- const [fx, fy] = this.jitter(x, y);
241
+ const [cx, cy] = this.clampToViewport(x, y);
242
+ const [fx, fy] = this.jitter(cx, cy);
230
243
  if (this._humanize?.mouseMovement) {
231
244
  await this.humanMouseMove(fx, fy);
232
245
  }
@@ -240,9 +253,10 @@ var BrowserProvider = class {
240
253
  await this.humanPause();
241
254
  }
242
255
  async clickAndType(x, y, text) {
243
- await this.click(x, y);
256
+ const [cx, cy] = this.clampToViewport(x, y);
257
+ await this.click(cx, cy);
244
258
  await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
245
- const [fx, fy] = this.jitter(x, y);
259
+ const [fx, fy] = this.jitter(cx, cy);
246
260
  await this.page.mouse.click(fx, fy, { clickCount: 3 });
247
261
  await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
248
262
  await this.type(text);
@@ -271,8 +285,10 @@ var BrowserProvider = class {
271
285
  // ── DOM Extraction ───────────────────────────────────────────────────
272
286
  async extractDOM(opts) {
273
287
  this.ensurePage();
274
- const max = opts?.maxElements ?? 80;
288
+ const max = opts?.maxElements ?? 120;
275
289
  const elements = await this.page.evaluate((limit) => {
290
+ const doc = globalThis.document;
291
+ const win = globalThis.window;
276
292
  const selectors = [
277
293
  "a[href]",
278
294
  "button",
@@ -283,37 +299,62 @@ var BrowserProvider = class {
283
299
  "[role='link']",
284
300
  "[role='tab']",
285
301
  "[role='menuitem']",
302
+ "[role='checkbox']",
303
+ "[role='radio']",
304
+ "[role='switch']",
305
+ "[role='combobox']",
306
+ "[role='option']",
286
307
  "[onclick]",
287
- "[contenteditable='true']"
308
+ "[contenteditable='true']",
309
+ "[tabindex]:not([tabindex='-1'])"
288
310
  ];
289
- const all = globalThis.document.querySelectorAll(selectors.join(","));
290
- const lines = [];
291
- let count = 0;
292
- const vh = globalThis.window.innerHeight;
293
- const vw = globalThis.window.innerWidth;
311
+ const all = Array.from(doc.querySelectorAll(selectors.join(",")));
312
+ const vh = win.innerHeight;
313
+ const vw = win.innerWidth;
314
+ const entries = [];
315
+ const seen = /* @__PURE__ */ new Set();
294
316
  for (const el of all) {
295
- if (count >= limit) break;
317
+ if (seen.has(el)) continue;
296
318
  const rect = el.getBoundingClientRect();
297
- if (rect.width === 0 || rect.height === 0) continue;
319
+ if (rect.width < 4 || rect.height < 4) continue;
298
320
  if (rect.bottom < 0 || rect.top > vh) continue;
299
321
  if (rect.right < 0 || rect.left > vw) continue;
322
+ const style = win.getComputedStyle(el);
323
+ if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
324
+ continue;
325
+ }
326
+ const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
327
+ const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
328
+ try {
329
+ const hit = doc.elementFromPoint(cx, cy);
330
+ if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
331
+ continue;
332
+ }
333
+ } catch {
334
+ }
335
+ seen.add(el);
300
336
  const tag = el.tagName.toLowerCase();
301
337
  const role = el.getAttribute("role") || tag;
302
338
  const type = el.getAttribute("type") || "";
303
- const text = (el.textContent || "").trim().slice(0, 80);
339
+ const innerText = el.innerText || el.textContent || "";
340
+ const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
304
341
  const placeholder = el.getAttribute("placeholder") || "";
305
342
  const ariaLabel = el.getAttribute("aria-label") || "";
343
+ const title = el.getAttribute("title") || "";
344
+ const name = el.getAttribute("name") || "";
306
345
  const href = el.getAttribute("href") || "";
307
346
  const value = el.value || "";
308
- const cx = Math.round(rect.left + rect.width / 2);
309
- const cy = Math.round(rect.top + rect.height / 2);
310
- let label = ariaLabel || text || placeholder || value;
347
+ let label = ariaLabel || text || placeholder || title || value || name;
311
348
  if (!label && href) label = href.slice(0, 60);
312
349
  if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
313
- lines.push(`[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`);
314
- count++;
350
+ entries.push({
351
+ cx,
352
+ cy,
353
+ line: `[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`
354
+ });
315
355
  }
316
- return lines.join("\n");
356
+ entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
357
+ return entries.slice(0, limit).map((e) => e.line).join("\n");
317
358
  }, max);
318
359
  return elements;
319
360
  }
@@ -421,6 +462,18 @@ var BrowserProvider = class {
421
462
  const j = this._humanize.clickJitter;
422
463
  return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
423
464
  }
465
+ /**
466
+ * Safety net: clamp coordinates returned by the vision model to the actual
467
+ * viewport. If a model occasionally returns image-space coordinates from a
468
+ * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
469
+ * from clicking at e.g. (2200, 1300) and either erroring or landing on a
470
+ * random off-screen element.
471
+ */
472
+ clampToViewport(x, y) {
473
+ const cx = Math.max(0, Math.min(this._viewport.width - 1, Math.round(x)));
474
+ const cy = Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)));
475
+ return [cx, cy];
476
+ }
424
477
  /**
425
478
  * Simulate human mouse movement using Bézier-like interpolation.
426
479
  * Moves from the current mouse position to the target in small steps.
@@ -464,8 +517,12 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
464
517
  const lines = [
465
518
  `You are a browser automation agent. You receive a screenshot of a web browser and decide what action to take next to complete the user's task.`,
466
519
  ``,
467
- `## Viewport`,
468
- `The browser viewport is ${viewport.width}\xD7${viewport.height} pixels. All coordinates you provide must be within this range.`,
520
+ `## Viewport & Coordinate System`,
521
+ `- The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels.`,
522
+ `- The screenshot you are given has EXACTLY the same dimensions: ${viewport.width}\xD7${viewport.height} pixels (top-left = 0,0).`,
523
+ `- ALL coordinates you return MUST be within 0..${viewport.width - 1} for x and 0..${viewport.height - 1} for y.`,
524
+ `- Coordinates outside this range will be rejected \u2014 your click will go nowhere.`,
525
+ `- If a "Interactive elements on page" list is provided below, PREFER its coordinates over your own visual estimation. They are exact element centers in the same coordinate system.`,
469
526
  ``,
470
527
  `## Available Actions`,
471
528
  `Respond with a JSON object containing one of these actions:`,
@@ -506,14 +563,15 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
506
563
  `## Rules`,
507
564
  `1. ALWAYS look at the screenshot carefully before deciding your action.`,
508
565
  `2. Provide coordinates that target the CENTER of the element you want to interact with.`,
509
- `3. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
510
- `4. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
511
- `5. If your previous action didn't produce the expected result, try a different approach.`,
512
- `6. If a page is loading or blank, use "wait" with a short delay and try again.`,
513
- `7. If you see a cookie banner or popup, dismiss it first before proceeding with the task.`,
514
- `8. NEVER hallucinate content. Only report what you can actually see on the screen.`,
515
- `9. When the task is fully complete, use "done" immediately with a comprehensive result.`,
516
- `10. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
566
+ `3. When an "Interactive elements on page" list is provided, identify the target element in that list by its visible label (button text, aria-label, placeholder, link href) and use ITS coordinates \u2014 they are the ground truth. Only fall back to visual estimation when no matching element is listed.`,
567
+ `4. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
568
+ `5. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
569
+ `6. If your previous click landed on the wrong element (e.g. page didn't change or a different element became focused), the target was probably misaligned \u2014 re-read the elements list and pick a different coordinate; do NOT repeat the same click.`,
570
+ `7. If a page is loading or blank, use "wait" with a short delay and try again.`,
571
+ `8. If you see a cookie banner, consent dialog, or popup, dismiss it FIRST before proceeding with the task \u2014 these often intercept clicks on elements behind them.`,
572
+ `9. NEVER hallucinate content. Only report what you can actually see on the screen.`,
573
+ `10. When the task is fully complete, use "done" immediately with a comprehensive result.`,
574
+ `11. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
517
575
  ``,
518
576
  `## Response Format`,
519
577
  `Respond with ONLY a valid JSON object. No markdown, no explanation, just the JSON action.`
@@ -619,7 +677,7 @@ var BrowserAgent = class {
619
677
  this.defaultStartUrl = config.startUrl;
620
678
  this.waitAfterAction = config.waitAfterAction ?? 1500;
621
679
  this.maxRepeats = config.maxRepeats ?? 3;
622
- this.useDOM = config.useDOM ?? false;
680
+ this.useDOM = config.useDOM ?? true;
623
681
  this.storageState = config.storageState;
624
682
  this.recordVideo = config.recordVideo;
625
683
  this.credentials = config.credentials;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agentium/browser",
3
- "version": "2.0.4",
3
+ "version": "2.0.6",
4
4
  "description": "Browser automation agent for Agentium using Playwright",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -46,7 +46,7 @@
46
46
  "typescript": "^5.6.0"
47
47
  },
48
48
  "peerDependencies": {
49
- "@agentium/core": "^2.0.4",
49
+ "@agentium/core": "^2.0.6",
50
50
  "playwright": ">=1.40.0"
51
51
  },
52
52
  "peerDependenciesMeta": {