@agent-browser-io/browser 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +70 -0
  2. package/bin/agent-browser-cli.cjs +2 -0
  3. package/bin/agent-browser-mcp.cjs +2 -0
  4. package/bin/index.cjs +54 -0
  5. package/dist/cjs/ai-sdk/createBrowserTools.d.ts +58 -0
  6. package/dist/cjs/ai-sdk/createBrowserTools.d.ts.map +1 -0
  7. package/dist/cjs/ai-sdk/createBrowserTools.js +192 -0
  8. package/dist/cjs/ai-sdk/createBrowserTools.js.map +1 -0
  9. package/dist/cjs/cli/cli.d.ts +6 -0
  10. package/dist/cjs/cli/cli.d.ts.map +1 -0
  11. package/dist/cjs/cli/cli.js +149 -0
  12. package/dist/cjs/cli/cli.js.map +1 -0
  13. package/dist/cjs/core/agent-browser/agent-browser.d.ts +51 -0
  14. package/dist/cjs/core/agent-browser/agent-browser.d.ts.map +1 -0
  15. package/dist/cjs/core/agent-browser/agent-browser.js +80 -0
  16. package/dist/cjs/core/agent-browser/agent-browser.js.map +1 -0
  17. package/dist/cjs/core/agent-browser/normalize-script.d.ts +3 -0
  18. package/dist/cjs/core/agent-browser/normalize-script.d.ts.map +1 -0
  19. package/dist/cjs/core/agent-browser/normalize-script.js +531 -0
  20. package/dist/cjs/core/agent-browser/normalize-script.js.map +1 -0
  21. package/dist/cjs/core/browser-backend/browser-backend.d.ts +23 -0
  22. package/dist/cjs/core/browser-backend/browser-backend.d.ts.map +1 -0
  23. package/dist/cjs/core/browser-backend/browser-backend.js +6 -0
  24. package/dist/cjs/core/browser-backend/browser-backend.js.map +1 -0
  25. package/dist/cjs/core/browser-backend/index.d.ts +3 -0
  26. package/dist/cjs/core/browser-backend/index.d.ts.map +1 -0
  27. package/dist/cjs/core/browser-backend/index.js +6 -0
  28. package/dist/cjs/core/browser-backend/index.js.map +1 -0
  29. package/dist/cjs/core/browser-backend/playwright-browser-backend.d.ts +27 -0
  30. package/dist/cjs/core/browser-backend/playwright-browser-backend.d.ts.map +1 -0
  31. package/dist/cjs/core/browser-backend/playwright-browser-backend.js +86 -0
  32. package/dist/cjs/core/browser-backend/playwright-browser-backend.js.map +1 -0
  33. package/dist/cjs/index.d.ts +5 -0
  34. package/dist/cjs/index.d.ts.map +1 -1
  35. package/dist/cjs/index.js +8 -3
  36. package/dist/cjs/index.js.map +1 -1
  37. package/dist/cjs/mcp/server.d.ts +5 -0
  38. package/dist/cjs/mcp/server.d.ts.map +1 -0
  39. package/dist/cjs/mcp/server.js +284 -0
  40. package/dist/cjs/mcp/server.js.map +1 -0
  41. package/dist/esm/ai-sdk/createBrowserTools.d.ts +58 -0
  42. package/dist/esm/ai-sdk/createBrowserTools.d.ts.map +1 -0
  43. package/dist/esm/ai-sdk/createBrowserTools.js +189 -0
  44. package/dist/esm/ai-sdk/createBrowserTools.js.map +1 -0
  45. package/dist/esm/cli/cli.d.ts +6 -0
  46. package/dist/esm/cli/cli.d.ts.map +1 -0
  47. package/dist/esm/cli/cli.js +114 -0
  48. package/dist/esm/cli/cli.js.map +1 -0
  49. package/dist/esm/core/agent-browser/agent-browser.d.ts +51 -0
  50. package/dist/esm/core/agent-browser/agent-browser.d.ts.map +1 -0
  51. package/dist/esm/core/agent-browser/agent-browser.js +76 -0
  52. package/dist/esm/core/agent-browser/agent-browser.js.map +1 -0
  53. package/dist/esm/core/agent-browser/normalize-script.d.ts +3 -0
  54. package/dist/esm/core/agent-browser/normalize-script.d.ts.map +1 -0
  55. package/dist/esm/core/agent-browser/normalize-script.js +528 -0
  56. package/dist/esm/core/agent-browser/normalize-script.js.map +1 -0
  57. package/dist/esm/core/browser-backend/browser-backend.d.ts +23 -0
  58. package/dist/esm/core/browser-backend/browser-backend.d.ts.map +1 -0
  59. package/dist/esm/core/browser-backend/browser-backend.js +5 -0
  60. package/dist/esm/core/browser-backend/browser-backend.js.map +1 -0
  61. package/dist/esm/core/browser-backend/index.d.ts +3 -0
  62. package/dist/esm/core/browser-backend/index.d.ts.map +1 -0
  63. package/dist/esm/core/browser-backend/index.js +2 -0
  64. package/dist/esm/core/browser-backend/index.js.map +1 -0
  65. package/dist/esm/core/browser-backend/playwright-browser-backend.d.ts +27 -0
  66. package/dist/esm/core/browser-backend/playwright-browser-backend.d.ts.map +1 -0
  67. package/dist/esm/core/browser-backend/playwright-browser-backend.js +82 -0
  68. package/dist/esm/core/browser-backend/playwright-browser-backend.js.map +1 -0
  69. package/dist/esm/index.d.ts +5 -0
  70. package/dist/esm/index.d.ts.map +1 -1
  71. package/dist/esm/index.js +4 -2
  72. package/dist/esm/index.js.map +1 -1
  73. package/dist/esm/mcp/server.d.ts +5 -0
  74. package/dist/esm/mcp/server.d.ts.map +1 -0
  75. package/dist/esm/mcp/server.js +282 -0
  76. package/dist/esm/mcp/server.js.map +1 -0
  77. package/package.json +14 -5
@@ -0,0 +1,531 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.NORMALIZE_SCRIPT = void 0;
4
+ /** Injected browser script for wireframe normalization. Exported as string for evaluation in the page. */
5
+ exports.NORMALIZE_SCRIPT = `
6
+ // This script is intended to be used in the browser.
7
+ // It is not intended to be used in node.
8
+ (function applyWireframeMode() {
9
+ // --- PART A: INJECT CSS STYLES (idempotent) ---
10
+ if (!document.getElementById("wf-normalize")) {
11
+ const cssStyles = \`
12
+ /* FORCE GLOBAL MONOSPACE & METRICS */
13
+ * {
14
+ font-family: "Courier New", Courier, monospace !important;
15
+ font-size: 12px !important;
16
+ font-weight: normal !important;
17
+ color: #000000 !important;
18
+ line-height: 18px !important;
19
+ letter-spacing: 0px !important;
20
+ box-shadow: none !important;
21
+ text-shadow: none !important;
22
+ border-radius: 0 !important;
23
+ transition: none !important;
24
+ }
25
+
26
+ /* HIGH CONTRAST THEME */
27
+ body {
28
+ background-color: #ffffff !important;
29
+ color: #000000 !important;
30
+ }
31
+
32
+ /* STRUCTURAL OUTLINES */
33
+ div, section, article, header, footer, nav, aside, main {
34
+ border: 1px dotted #cccccc !important;
35
+ }
36
+
37
+ /* KILL PSEUDO-ELEMENT BACKGROUNDS */
38
+ *::before, *::after {
39
+ background-image: none !important;
40
+ background-color: transparent !important;
41
+ border: none !important;
42
+ }
43
+
44
+ /* INTERACTIVE ELEMENTS */
45
+ a, button, input[type="submit"], input[type="button"], [role="button"] {
46
+ border: 2px solid #000000 !important;
47
+ background-color: #ffffff !important;
48
+ color: #000000 !important;
49
+ text-transform: uppercase !important;
50
+ text-decoration: none !important;
51
+ font-weight: bold !important;
52
+ }
53
+
54
+ /* FORM INPUTS */
55
+ input, textarea, select {
56
+ border: 1px solid #000000 !important;
57
+ background-color: #ffffff !important;
58
+ color: #000000 !important;
59
+ font-family: monospace !important;
60
+ }
61
+
62
+ /* HIDE DECORATIVE MEDIA */
63
+ img, video, canvas, svg {
64
+ opacity: 0.5 !important;
65
+ filter: grayscale(100%) !important;
66
+ border: 1px dashed #000 !important;
67
+ }
68
+
69
+ /* BACKGROUND IMAGE INDICATORS */
70
+ [data-bg-image="true"] {
71
+ background-color: #f4f4f4 !important;
72
+ border: 1px dashed #555555 !important;
73
+ position: relative !important;
74
+ }
75
+ [data-bg-image="true"]::before {
76
+ content: "[ BG IMAGE ]";
77
+ position: absolute;
78
+ top: 0; right: 0;
79
+ background: #000000; color: #ffffff;
80
+ font-size: 10px !important;
81
+ padding: 2px 4px;
82
+ opacity: 0.8;
83
+ z-index: 9999;
84
+ pointer-events: none;
85
+ font-weight: normal !important;
86
+ }
87
+ \`;
88
+
89
+ const styleSheet = document.createElement("style");
90
+ styleSheet.id = "wf-normalize";
91
+ styleSheet.type = "text/css";
92
+ styleSheet.innerText = cssStyles;
93
+ document.head.appendChild(styleSheet);
94
+ }
95
+
96
+ // --- PART B: NORMALIZE DOM CONTENT (idempotent) ---
97
+ if (!document.body) return; // page not ready
98
+
99
+ // Helper: check if element is the top-most (not behind an overlay) at its center
100
+ function isElementVisible(el) {
101
+ var rect = el.getBoundingClientRect();
102
+ var cx = rect.left + rect.width / 2;
103
+ var cy = rect.top + rect.height / 2;
104
+ if (cx < 0 || cy < 0 || cx >= window.innerWidth || cy >= window.innerHeight) return false;
105
+ var topEl = document.elementFromPoint(cx, cy);
106
+ if (!topEl) return false;
107
+ return el.contains(topEl) || topEl.contains(el);
108
+ }
109
+
110
+ // Remove old ref IDs and reset counter
111
+ document.querySelectorAll("[data-ref-id]").forEach(function(el) {
112
+ el.removeAttribute("data-ref-id");
113
+ });
114
+ var refCounter = 1;
115
+
116
+ // Only run DOM mutations (media replacement, accessibility labels, bg stripping) once
117
+ if (!document.body.hasAttribute("data-wf-normalized")) {
118
+ document.body.setAttribute("data-wf-normalized", "true");
119
+
120
+ // 1. REVEAL ACCESSIBILITY LABELS (ICON BUTTONS)
121
+ var interactives = document.querySelectorAll('a, button, [role="button"]');
122
+ interactives.forEach(function(el) {
123
+ if (el.innerText.trim() === '') {
124
+ var label = el.getAttribute('aria-label') || el.getAttribute('title');
125
+ // Also check child elements for hints (e.g. HN vote arrows: <a><div title="upvote"></a>)
126
+ if (!label) {
127
+ var hintEl = el.querySelector('[aria-label], [title]');
128
+ if (hintEl) label = hintEl.getAttribute('aria-label') || hintEl.getAttribute('title');
129
+ }
130
+ if (label) {
131
+ el.innerText = label;
132
+ el.style.display = 'inline-block';
133
+ el.style.width = 'auto';
134
+ el.style.height = 'auto';
135
+ }
136
+ }
137
+ });
138
+
139
+ // 2. REPLACE MEDIA WITH INTELLIGENT PLACEHOLDERS
140
+ var mediaElements = document.querySelectorAll('img, svg, video, canvas');
141
+ mediaElements.forEach(function(el) {
142
+ var rect = el.getBoundingClientRect();
143
+ if (rect.width < 10 || rect.height < 10) return;
144
+
145
+ var placeholder = document.createElement('div');
146
+ placeholder.style.cssText =
147
+ 'width:' + rect.width + 'px;' +
148
+ 'height:' + rect.height + 'px;' +
149
+ 'background:#f0f0f0;color:#000;font-family:monospace;font-weight:bold;' +
150
+ 'display:flex;align-items:center;justify-content:center;text-align:center;' +
151
+ 'overflow:hidden;box-sizing:border-box;';
152
+
153
+ var altText = el.getAttribute('alt') ||
154
+ el.getAttribute('title') ||
155
+ el.getAttribute('aria-label');
156
+
157
+ if (!altText && el.tagName.toLowerCase() === 'svg') {
158
+ var titleEl = el.querySelector('title');
159
+ if (titleEl) altText = titleEl.textContent;
160
+ }
161
+
162
+ if (rect.width <= 50 && rect.height <= 50) {
163
+ placeholder.style.border = '1px solid #333';
164
+ placeholder.style.fontSize = '8px';
165
+ placeholder.style.padding = '0';
166
+ // Use short label for tiny icons to avoid overflow
167
+ if (rect.width <= 20 || rect.height <= 20) {
168
+ placeholder.innerText = altText ? altText.substring(0, 1) : '*';
169
+ } else {
170
+ var iconLabel = altText ? altText.substring(0, 8) : "ICON";
171
+ placeholder.innerText = '[' + iconLabel + ']';
172
+ }
173
+ } else {
174
+ placeholder.style.border = '2px solid #000';
175
+ placeholder.style.fontSize = '10px';
176
+ placeholder.style.padding = '2px';
177
+ placeholder.style.wordBreak = 'break-word';
178
+ var imgLabel = altText ? altText : "IMAGE";
179
+ placeholder.innerText = '[' + imgLabel + ']';
180
+ }
181
+
182
+ if (el.parentNode) {
183
+ el.parentNode.replaceChild(placeholder, el);
184
+ }
185
+ });
186
+
187
+ // 3. CONVERT BACKGROUND IMAGES & STRIP COLORS
188
+ var allElements = document.querySelectorAll('*');
189
+ allElements.forEach(function(el) {
190
+ var style = window.getComputedStyle(el);
191
+ var hasBgImage = style.backgroundImage !== 'none' && style.backgroundImage !== '';
192
+ var hasBgColor = style.backgroundColor !== 'rgba(0, 0, 0, 0)' &&
193
+ style.backgroundColor !== 'transparent' &&
194
+ style.backgroundColor !== 'rgb(255, 255, 255)';
195
+
196
+ if (hasBgImage) {
197
+ el.style.backgroundImage = 'none';
198
+ el.setAttribute('data-bg-image', 'true');
199
+ }
200
+
201
+ if (hasBgColor) {
202
+ el.style.backgroundColor = '#ffffff';
203
+ }
204
+ });
205
+ }
206
+
207
+ // 4. REMOVE TRANSPARENT OVERLAY BACKDROPS (run every time — new overlays can appear)
208
+ // Full-viewport high z-index divs with no meaningful text are just backdrop overlays
209
+ // that block elementFromPoint for content behind them
210
+ var overlayDivs = document.querySelectorAll('div');
211
+ overlayDivs.forEach(function(el) {
212
+ var style = window.getComputedStyle(el);
213
+ var z = parseInt(style.zIndex) || 0;
214
+ if (z < 100 || style.display === 'none') return;
215
+ var rect = el.getBoundingClientRect();
216
+ if (rect.width < window.innerWidth * 0.9 || rect.height < window.innerHeight * 0.9) return;
217
+ var text = (el.innerText || '').trim();
218
+ if (text.length < 5) {
219
+ el.style.display = 'none';
220
+ }
221
+ });
222
+
223
+ // Overflow buffer: capture high-z-index elements pushed below viewport by CSS normalization
224
+ var OVERFLOW_BUFFER = 200;
225
+ var captureHeight = window.innerHeight + OVERFLOW_BUFFER;
226
+
227
+ // 5. TAG INTERACTIVE ELEMENTS (always re-run since we cleared ref IDs)
228
+ // Only tag elements visible, in viewport (+ overflow buffer), and not behind overlays
229
+ function tagIfVisible(el) {
230
+ if (el.hasAttribute('data-ref-id')) return; // already tagged
231
+ var rect = el.getBoundingClientRect();
232
+ var style = window.getComputedStyle(el);
233
+ if (rect.width === 0 || rect.height === 0 || style.display === 'none' || style.visibility === 'hidden') return;
234
+ if (rect.bottom < 0 || rect.top >= captureHeight || rect.right < 0 || rect.left >= window.innerWidth) return;
235
+ // Elements in overflow zone (below viewport): can't use elementFromPoint, just tag them
236
+ if (rect.top >= window.innerHeight) {
237
+ el.setAttribute('data-ref-id', String(refCounter++));
238
+ return;
239
+ }
240
+ if (!isElementVisible(el)) return;
241
+ el.setAttribute('data-ref-id', String(refCounter++));
242
+ }
243
+
244
+ // Pass 1: semantic interactive elements
245
+ var allInteractives = document.querySelectorAll('a, button, input, textarea, select, [role="button"], [onclick], [tabindex="0"]');
246
+ allInteractives.forEach(tagIfVisible);
247
+
248
+ // Pass 2: non-semantic clickable elements (divs/spans with cursor:pointer and text)
249
+ // This catches sites that use <div> as buttons without proper ARIA roles
250
+ var potentialClickables = document.querySelectorAll('div, span');
251
+ potentialClickables.forEach(function(el) {
252
+ if (el.hasAttribute('data-ref-id')) return;
253
+ // Skip if already inside a tagged interactive element
254
+ if (el.closest && el.closest('[data-ref-id]')) return;
255
+ var style = window.getComputedStyle(el);
256
+ if (style.cursor !== 'pointer') return;
257
+ // Must have direct text content (not just children with text)
258
+ var hasDirectText = false;
259
+ for (var i = 0; i < el.childNodes.length; i++) {
260
+ if (el.childNodes[i].nodeType === 3 && el.childNodes[i].textContent.trim()) {
261
+ hasDirectText = true;
262
+ break;
263
+ }
264
+ }
265
+ if (!hasDirectText) return;
266
+ tagIfVisible(el);
267
+ });
268
+
269
+ console.log("Normalization Complete.");
270
+
271
+ // --- PART C: WIREFRAME STRING GENERATOR ---
272
+ function generateWireframeString() {
273
+ // 1. Measure CHAR_W dynamically
274
+ var probe = document.createElement('span');
275
+ probe.style.cssText = 'font-family:"Courier New",Courier,monospace;font-size:12px;line-height:18px;letter-spacing:0px;position:absolute;top:-9999px;left:-9999px;white-space:pre;visibility:hidden;';
276
+ probe.textContent = 'MMMMMMMMMM';
277
+ document.body.appendChild(probe);
278
+ var CHAR_W = probe.getBoundingClientRect().width / 10;
279
+ document.body.removeChild(probe);
280
+ if (CHAR_W <= 0) CHAR_W = 7.2; // fallback
281
+
282
+ var CHAR_H = 18;
283
+
284
+ // 2. Create grid (includes overflow buffer for elements pushed below viewport)
285
+ var OVERFLOW_BUFFER_C = 200;
286
+ var captureHeightC = window.innerHeight + OVERFLOW_BUFFER_C;
287
+ var gridWidth = Math.ceil(window.innerWidth / CHAR_W);
288
+ var gridHeight = Math.ceil(captureHeightC / CHAR_H);
289
+ var grid = [];
290
+ for (var r = 0; r < gridHeight; r++) {
291
+ var row = [];
292
+ for (var c = 0; c < gridWidth; c++) {
293
+ row.push(' ');
294
+ }
295
+ grid.push(row);
296
+ }
297
+
298
+ function writeToGrid(x, y, str) {
299
+ if (y < 0 || y >= gridHeight) return;
300
+ for (var i = 0; i < str.length; i++) {
301
+ var curX = x + i;
302
+ if (curX >= 0 && curX < gridWidth) {
303
+ grid[y][curX] = str[i];
304
+ }
305
+ }
306
+ }
307
+
308
+ // 3. Draw borders only for block-level interactive elements (buttons, form controls)
309
+ // Skip inline links — they just get ref labels.
310
+ var refElements = document.querySelectorAll('[data-ref-id]');
311
+ refElements.forEach(function(el) {
312
+ var rect = el.getBoundingClientRect();
313
+ var style = window.getComputedStyle(el);
314
+ if (style.display === 'none' || style.visibility === 'hidden' || rect.width === 0 || rect.height === 0) return;
315
+ // Skip elements outside capture area
316
+ if (rect.bottom < 0 || rect.top >= captureHeightC || rect.right < 0 || rect.left >= window.innerWidth) return;
317
+
318
+ // Only draw borders for buttons, form elements, and block-level role=button
319
+ var tag = el.tagName;
320
+ var isFormEl = (tag === 'BUTTON' || tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT');
321
+ var isBlockButton = (el.getAttribute('role') === 'button' && style.display !== 'inline');
322
+ if (!isFormEl && !isBlockButton) return;
323
+
324
+ var x = Math.floor(rect.left / CHAR_W);
325
+ var y = Math.floor(rect.top / CHAR_H);
326
+ var w = Math.max(2, Math.ceil(rect.width / CHAR_W));
327
+ var h = Math.max(2, Math.ceil(rect.height / CHAR_H));
328
+
329
+ // Top border
330
+ writeToGrid(x, y, '+' + repeat('-', Math.max(0, w - 2)) + '+');
331
+ // Side borders
332
+ for (var i = 1; i < h - 1; i++) {
333
+ writeToGrid(x, y + i, '|');
334
+ writeToGrid(x + w - 1, y + i, '|');
335
+ }
336
+ // Bottom border
337
+ if (h > 1) {
338
+ writeToGrid(x, y + h - 1, '+' + repeat('-', Math.max(0, w - 2)) + '+');
339
+ }
340
+ });
341
+
342
+ // Helper: check if element is the top-most at a given point
343
+ function isVisibleAt(el, px, py) {
344
+ if (px < 0 || py < 0 || px >= window.innerWidth) return false;
345
+ // Overflow zone: elementFromPoint doesn't work below viewport, assume visible
346
+ if (py >= window.innerHeight) return py < captureHeightC;
347
+ var topEl = document.elementFromPoint(px, py);
348
+ if (!topEl) return false;
349
+ return el.contains(topEl) || topEl.contains(el);
350
+ }
351
+
352
+ // 4. Render text via TreeWalker (text only, no labels yet)
353
+ var labeledRefs = {}; // refId -> {row, col} where the element's first text starts
354
+ var walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null, false);
355
+ var textNode;
356
+ while ((textNode = walker.nextNode())) {
357
+ var parent = textNode.parentElement;
358
+ if (!parent) continue;
359
+
360
+ // Skip script/style
361
+ var tag = parent.tagName;
362
+ if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'NOSCRIPT') continue;
363
+
364
+ // Skip invisible parents
365
+ var pStyle = window.getComputedStyle(parent);
366
+ if (pStyle.display === 'none' || pStyle.visibility === 'hidden' || pStyle.opacity === '0') continue;
367
+
368
+ // Skip empty text
369
+ var rawText = textNode.textContent;
370
+ if (!rawText || !rawText.trim()) continue;
371
+
372
+ // Skip form elements (handled separately)
373
+ if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || tag === 'OPTION') continue;
374
+
375
+ // Collapse whitespace
376
+ var collapsed = rawText.replace(/\\s+/g, ' ').trim();
377
+ if (!collapsed) continue;
378
+
379
+ // Track first text position for ref label insertion
380
+ var refAncestor = parent.closest ? parent.closest('[data-ref-id]') : null;
381
+
382
+ // Use Range + getClientRects for accurate per-line positioning
383
+ var range = document.createRange();
384
+ range.selectNodeContents(textNode);
385
+ var rects = range.getClientRects();
386
+
387
+ if (rects.length === 0) continue;
388
+
389
+ var charOffset = 0;
390
+ var firstWritten = false;
391
+ for (var ri = 0; ri < rects.length; ri++) {
392
+ var rr = rects[ri];
393
+ if (rr.width < 1 || rr.height < 1) continue;
394
+
395
+ // Skip rects outside capture area or covered by overlays
396
+ if (rr.bottom < 0 || rr.top >= captureHeightC || rr.right < 0 || rr.left >= window.innerWidth) {
397
+ charOffset += Math.max(1, Math.ceil(rr.width / CHAR_W));
398
+ continue;
399
+ }
400
+ var rcx = rr.left + rr.width / 2;
401
+ var rcy = rr.top + rr.height / 2;
402
+ if (!isVisibleAt(parent, rcx, rcy)) {
403
+ charOffset += Math.max(1, Math.ceil(rr.width / CHAR_W));
404
+ continue;
405
+ }
406
+
407
+ var gx = Math.floor(rr.left / CHAR_W);
408
+ var gy = Math.floor(rr.top / CHAR_H);
409
+ var charsInLine = Math.max(1, Math.ceil(rr.width / CHAR_W));
410
+ // Cap to actual remaining text length
411
+ var remaining = collapsed.length - charOffset;
412
+ if (charsInLine > remaining) charsInLine = remaining;
413
+ var slice = collapsed.substring(charOffset, charOffset + charsInLine);
414
+ if (slice) {
415
+ writeToGrid(gx, gy, slice);
416
+
417
+ // Record insertion point for this ref's label
418
+ if (!firstWritten && refAncestor) {
419
+ var rid = refAncestor.getAttribute('data-ref-id');
420
+ if (!labeledRefs[rid]) {
421
+ labeledRefs[rid] = {row: gy, col: gx};
422
+ }
423
+ firstWritten = true;
424
+ }
425
+ }
426
+ charOffset += charsInLine;
427
+ }
428
+ }
429
+
430
+ // 5. Handle form inputs
431
+ var formEls = document.querySelectorAll('input, textarea, select');
432
+ formEls.forEach(function(el) {
433
+ var rect = el.getBoundingClientRect();
434
+ var style = window.getComputedStyle(el);
435
+ if (style.display === 'none' || style.visibility === 'hidden' || rect.width === 0 || rect.height === 0) return;
436
+
437
+ var x = Math.floor(rect.left / CHAR_W);
438
+ var y = Math.floor(rect.top / CHAR_H);
439
+ var w = Math.max(2, Math.ceil(rect.width / CHAR_W));
440
+ var h = Math.max(2, Math.ceil(rect.height / CHAR_H));
441
+ var midY = y + Math.floor((h - 1) / 2);
442
+ var refId = el.getAttribute('data-ref-id');
443
+
444
+ if (el.tagName === 'SELECT') {
445
+ var selectedText = el.options && el.selectedIndex >= 0 ? el.options[el.selectedIndex].text : '';
446
+ var selectContent = selectedText.substring(0, Math.max(0, w - 4));
447
+ if (selectContent) writeToGrid(x + 1, midY, selectContent);
448
+ writeToGrid(x + w - 2, midY, 'v');
449
+ } else {
450
+ var val = el.value || el.getAttribute('placeholder') || '';
451
+ if (val) writeToGrid(x + 1, midY, val);
452
+ }
453
+
454
+ // Track form element position for label insertion
455
+ if (refId && !labeledRefs[refId]) {
456
+ labeledRefs[refId] = {row: midY, col: x};
457
+ }
458
+ });
459
+
460
+ // Also track ref elements with no text at all (empty buttons, icons)
461
+ refElements.forEach(function(el) {
462
+ var refId = el.getAttribute('data-ref-id');
463
+ if (labeledRefs[refId]) return;
464
+ var rect = el.getBoundingClientRect();
465
+ var style = window.getComputedStyle(el);
466
+ if (style.display === 'none' || style.visibility === 'hidden' || rect.width === 0 || rect.height === 0) return;
467
+ labeledRefs[refId] = {row: Math.floor(rect.top / CHAR_H), col: Math.floor(rect.left / CHAR_W)};
468
+ });
469
+
470
+ // 6. Insert ref labels by splicing into rows (never overwrites text)
471
+ // Collect all labels per row, sorted right-to-left so insertions don't shift later positions
472
+ var rowInsertions = {}; // row -> [{col, label}]
473
+ refElements.forEach(function(el) {
474
+ var refId = el.getAttribute('data-ref-id');
475
+ if (!refId) return;
476
+ var pos = labeledRefs[refId];
477
+ if (!pos) return;
478
+ var label = '[' + refId + ']';
479
+ if (!rowInsertions[pos.row]) rowInsertions[pos.row] = [];
480
+ rowInsertions[pos.row].push({col: pos.col, label: label});
481
+ });
482
+
483
+ // For each row with insertions, splice labels into the row string
484
+ for (var rowIdx in rowInsertions) {
485
+ var r = parseInt(rowIdx, 10);
486
+ if (r < 0 || r >= gridHeight) continue;
487
+ var inserts = rowInsertions[r];
488
+ // Sort by column descending so we insert right-to-left (preserves earlier positions)
489
+ inserts.sort(function(a, b) { return b.col - a.col; });
490
+ // Convert row to string, splice in labels, write back
491
+ var rowStr = grid[r].join('');
492
+ for (var ii = 0; ii < inserts.length; ii++) {
493
+ var ins = inserts[ii];
494
+ var c = Math.max(0, Math.min(ins.col, rowStr.length));
495
+ // Avoid splitting inside a [...] bracket sequence — shift to before the '['
496
+ var openBracket = rowStr.lastIndexOf('[', c);
497
+ if (openBracket >= 0) {
498
+ var closeBracket = rowStr.indexOf(']', openBracket);
499
+ if (closeBracket >= c) {
500
+ // Insertion point is inside [..], shift to before the [
501
+ c = openBracket;
502
+ }
503
+ }
504
+ rowStr = rowStr.substring(0, c) + ins.label + rowStr.substring(c);
505
+ }
506
+ // Write back to grid as variable-width row (will be joined as string in step 7)
507
+ grid[r] = rowStr.split('');
508
+ }
509
+
510
+ // 7. Trim output
511
+ var lines = [];
512
+ for (var r = 0; r < gridHeight; r++) {
513
+ lines.push(grid[r].join('').replace(/\\s+$/, ''));
514
+ }
515
+ // Drop trailing empty rows
516
+ while (lines.length > 0 && lines[lines.length - 1] === '') {
517
+ lines.pop();
518
+ }
519
+ return lines.join('\\n');
520
+ }
521
+
522
+ function repeat(ch, count) {
523
+ var s = '';
524
+ for (var i = 0; i < count; i++) s += ch;
525
+ return s;
526
+ }
527
+
528
+ window.generateWireframeString = generateWireframeString;
529
+ })();
530
+ `;
531
+ //# sourceMappingURL=normalize-script.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalize-script.js","sourceRoot":"","sources":["../../../../src/core/agent-browser/normalize-script.ts"],"names":[],"mappings":";;;AAAA,0GAA0G;AAC7F,QAAA,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6gB/B,CAAC"}
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Browser interface for agent-browser-io
3
+ */
4
+ export interface IBrowserBackend {
5
+ launch(): Promise<void>;
6
+ navigate(url: string): Promise<void>;
7
+ click(selector: string): Promise<void>;
8
+ type(selector: string, text: string): Promise<void>;
9
+ evaluate(script: string): Promise<any>;
10
+ dblclick(selector: string): Promise<void>;
11
+ fill(selector: string, text: string): Promise<void>;
12
+ press(key: string): Promise<void>;
13
+ hover(selector: string): Promise<void>;
14
+ select(selector: string, value: string): Promise<void>;
15
+ check(selector: string): Promise<void>;
16
+ uncheck(selector: string): Promise<void>;
17
+ scroll(direction: 'up' | 'down' | 'left' | 'right', pixels?: number): Promise<void>;
18
+ screenshot(path?: string, options?: {
19
+ fullPage?: boolean;
20
+ }): Promise<Buffer | void>;
21
+ close(): Promise<void>;
22
+ }
23
+ //# sourceMappingURL=browser-backend.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser-backend.d.ts","sourceRoot":"","sources":["../../../../src/core/browser-backend/browser-backend.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,eAAe;IAC5B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACvC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpF,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACpF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B"}
@@ -0,0 +1,6 @@
1
+ "use strict";
2
+ /**
3
+ * Browser interface for agent-browser-io
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ //# sourceMappingURL=browser-backend.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser-backend.js","sourceRoot":"","sources":["../../../../src/core/browser-backend/browser-backend.ts"],"names":[],"mappings":";AAAA;;GAEG"}
@@ -0,0 +1,3 @@
1
+ export type { IBrowserBackend } from './browser-backend.js';
2
+ export { PlaywrightBrowserBackend as DefaultBrowserBackend } from './playwright-browser-backend.js';
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/core/browser-backend/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,wBAAwB,IAAI,qBAAqB,EAAE,MAAM,iCAAiC,CAAC"}
@@ -0,0 +1,6 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.DefaultBrowserBackend = void 0;
4
+ var playwright_browser_backend_js_1 = require("./playwright-browser-backend.js");
5
+ Object.defineProperty(exports, "DefaultBrowserBackend", { enumerable: true, get: function () { return playwright_browser_backend_js_1.PlaywrightBrowserBackend; } });
6
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/core/browser-backend/index.ts"],"names":[],"mappings":";;;AACA,iFAAoG;AAA3F,sIAAA,wBAAwB,OAAyB"}
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Default browser implementation using Playwright
3
+ */
4
+ import type { IBrowserBackend } from './browser-backend.js';
5
+ export declare class PlaywrightBrowserBackend implements IBrowserBackend {
6
+ private browser;
7
+ private page;
8
+ launch(): Promise<void>;
9
+ navigate(url: string): Promise<void>;
10
+ click(selector: string): Promise<void>;
11
+ type(selector: string, text: string): Promise<void>;
12
+ evaluate(script: string): Promise<unknown>;
13
+ private ensurePage;
14
+ dblclick(selector: string): Promise<void>;
15
+ fill(selector: string, text: string): Promise<void>;
16
+ press(key: string): Promise<void>;
17
+ hover(selector: string): Promise<void>;
18
+ select(selector: string, value: string): Promise<void>;
19
+ check(selector: string): Promise<void>;
20
+ uncheck(selector: string): Promise<void>;
21
+ scroll(direction: 'up' | 'down' | 'left' | 'right', pixels?: number): Promise<void>;
22
+ screenshot(path?: string, options?: {
23
+ fullPage?: boolean;
24
+ }): Promise<Buffer | void>;
25
+ close(): Promise<void>;
26
+ }
27
+ //# sourceMappingURL=playwright-browser-backend.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"playwright-browser-backend.d.ts","sourceRoot":"","sources":["../../../../src/core/browser-backend/playwright-browser-backend.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5D,qBAAa,wBAAyB,YAAW,eAAe;IAC5D,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,IAAI,CAAqB;IAE3B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAKvB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;YAKlC,UAAU;IAMlB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKzC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKnD,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,MAAM,SAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOhF,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAMnF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAO/B"}