@browserless/agent 14.0.2 → 14.0.4

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/agent",
   "description": "Drive a Puppeteer page toward a natural-language goal using finite decisions.",
   "homepage": "https://browserless.js.org",
-  "version": "14.0.2",
+  "version": "14.0.4",
   "main": "src",
   "author": {
     "email": "hello@microlink.io",
@@ -59,5 +59,5 @@
     "access": "public"
   },
   "types": "src/index.d.ts",
-  "gitHead": "2f8b09a1509de4868c5ca2981ed147af8638968d"
+  "gitHead": "bf9a31cbe256022543902ec9e748dec6b5abf06b"
 }

package/src/model.js

@@ -345,7 +345,14 @@ const fieldText = async (goal, action, state, history, model, options) => {
         system: TEXT_VALUE,
         prompt: JSON.stringify({
           goal,
-          field: { label: action.label, role: action.role, value: action.value },
+          field: {
+            id: action.id,
+            node: action.node,
+            label: action.label,
+            role: action.role,
+            value: action.value,
+            context: state.scopes?.[action.node] ?? ''
+          },
           page: { title: state.title, text: state.text },
           recent_actions: history.slice(-TEXT_HISTORY_LENGTH)
         }),

package/src/snapshot.js

@@ -1,4 +1,4 @@
-/* global location, scrollX, scrollY, innerWidth, innerHeight */
+/* global HTMLElement, getComputedStyle, location, scrollX, scrollY, innerWidth, innerHeight */
 // Adapted from browser-use/jev-ultrafast snapshot.js (MIT). See README.md.
 module.exports = function snapshot () {
   return (() => {
@@ -169,6 +169,8 @@ module.exports = function snapshot () {
         .filter(safe)
         .map(e => [identity(e), e.value, e.checked, e.selectedIndex, e.disabled, e.readOnly])
     ]
+    // A named form control shadows the form's own innerText property.
+    const readInnerText = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'innerText').get
     cache.guard = e => {
       if (!e?.isConnected || !visibleNow(e)) return null
       const scope =
@@ -195,7 +197,9 @@ module.exports = function snapshot () {
             ])
           : null,
         e.getAttribute('href'),
-        scope?.innerText?.slice(0, 6000) || '',
+        (scope instanceof HTMLElement
+          ? readInnerText.call(scope).slice(0, 6000)
+          : scope?.textContent?.slice(0, 6000)) || '',
         [...(cache.closest(e, 'form')?.querySelectorAll('input,textarea,select') || [])]
           .filter(safe)
           .map(field => [
@@ -279,6 +283,74 @@ module.exports = function snapshot () {
         }
       }
     }
+    const collected = new Set(actions.map(action => cache.nodes.get(action.node)))
+    const listSelector = 'ul,ol,[role="list"],[role="listbox"],[role="menu"],[role="tablist"]'
+    const pointer = element => {
+      if (getComputedStyle(element).cursor !== 'pointer') return false
+      const parent = flatParent(element)
+      if (!parent || getComputedStyle(parent).cursor !== 'pointer') return true
+      return parent.matches(listSelector)
+    }
+    const inLayer = element => {
+      for (let current = element; current; current = flatParent(current)) {
+        if (current === document.body || current === document.documentElement) return false
+        const position = getComputedStyle(current).position
+        if (position === 'absolute' || position === 'fixed' || position === 'sticky') return true
+      }
+      return false
+    }
+    const pointerLists = new WeakMap()
+    const inPointerList = element => {
+      const parent = flatParent(element)
+      if (!parent?.matches(listSelector)) return false
+      let eligible = pointerLists.get(parent)
+      if (eligible == null) {
+        eligible = [...parent.children].filter(pointer).length >= 2
+        pointerLists.set(parent, eligible)
+      }
+      return eligible
+    }
+    const candidates = []
+    for (const element of queryAll('li,div,span,td,dd,p')) {
+      if (collected.has(element) || !visible(element) || !pointer(element)) continue
+      if (element.matches(':disabled') || cache.closest(element, '[aria-disabled="true"]')) continue
+      if (element.querySelector(selector)) continue
+      const rect = element.getBoundingClientRect()
+      const x = rect.x + rect.width / 2
+      const y = rect.y + rect.height / 2
+      if (
+        rect.width <= 0 ||
+        rect.height <= 0 ||
+        x < 0 ||
+        y < 0 ||
+        x >= innerWidth ||
+        y >= innerHeight
+      ) {
+        continue
+      }
+      const hit = cache.elementFromPoint(x, y)
+      if (hit && !cache.contains(element, hit)) continue
+      if (!inLayer(element) && !inPointerList(element)) continue
+      const label = (name(element) || '').replace(/\s+/g, ' ').trim().slice(0, 120)
+      if (!label) continue
+      candidates.push({ element, rect, label })
+    }
+    const rows = candidates.filter(
+      ({ element }) =>
+        !candidates.some(
+          other => other.element !== element && cache.contains(element, other.element)
+        )
+    )
+    for (const { element, rect, label } of rows.slice(0, 40)) {
+      actions.push({
+        node: identity(element),
+        role: 'button',
+        label,
+        rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
+        kind: 'click',
+        value: ''
+      })
+    }
     const words = []
     const range = document.createRange()
     let length = 0
@@ -350,8 +422,18 @@ module.exports = function snapshot () {
     const height = document.documentElement.scrollHeight
     const pageKey = cache.pageKey()
     const guards = {}
+    const scopes = {}
     for (const a of actions) {
-      if (!(a.node in guards)) guards[a.node] = cache.guard(cache.nodes.get(a.node))
+      if (a.node in guards) continue
+      const element = cache.nodes.get(a.node)
+      guards[a.node] = cache.guard(element)
+      const scope =
+        cache.closest(element, 'form,dialog,[role="dialog"],article,li,tr,[role="row"]') ||
+        flatParent(element)
+      scopes[a.node] =
+        (scope instanceof HTMLElement
+          ? readInnerText.call(scope).slice(0, 1000)
+          : scope?.textContent?.slice(0, 1000)) || ''
     }
     // Compare meaning and identity. Geometry is always resolved and hit-tested just before input.
     const semantics = actions.map(({ rect, ...action }) => action)
@@ -391,6 +473,7 @@ module.exports = function snapshot () {
       marker,
       pageKey,
       guards,
+      scopes,
       omittedActions
     }
   })()