@browserless/agent 14.0.3 → 14.0.5

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/agent",
   "description": "Drive a Puppeteer page toward a natural-language goal using finite decisions.",
   "homepage": "https://browserless.js.org",
-  "version": "14.0.3",
+  "version": "14.0.5",
   "main": "src",
   "author": {
     "email": "hello@microlink.io",
@@ -59,5 +59,5 @@
     "access": "public"
   },
   "types": "src/index.d.ts",
-  "gitHead": "f26f8ff96f57c2f2a848837097b560bd25ddd78a"
+  "gitHead": "6e8791c6a9061de43b9fc9e9b8e483bcfb970ef2"
 }

package/src/model.js

@@ -94,7 +94,7 @@ const actionSpace = actions => {
       indices.set(action.node, index)
       elements.push({
         index,
-        label: action.label.split(' → ')[0],
+        label: action.kind === 'select' ? action.label.split(' → ')[0] : action.label,
         role: action.role,
         value: action.current_value ?? action.value,
         checked: action.checked,
@@ -345,7 +345,14 @@ const fieldText = async (goal, action, state, history, model, options) => {
         system: TEXT_VALUE,
         prompt: JSON.stringify({
           goal,
-          field: { label: action.label, role: action.role, value: action.value },
+          field: {
+            id: action.id,
+            node: action.node,
+            label: action.label,
+            role: action.role,
+            value: action.value,
+            context: state.scopes?.[action.node] ?? ''
+          },
           page: { title: state.title, text: state.text },
           recent_actions: history.slice(-TEXT_HISTORY_LENGTH)
         }),

package/src/snapshot.js

@@ -1,4 +1,4 @@
-/* global HTMLElement, location, scrollX, scrollY, innerWidth, innerHeight */
+/* global HTMLElement, getComputedStyle, location, scrollX, scrollY, innerWidth, innerHeight */
 // Adapted from browser-use/jev-ultrafast snapshot.js (MIT). See README.md.
 module.exports = function snapshot () {
   return (() => {
@@ -102,7 +102,7 @@ module.exports = function snapshot () {
           .map(l => name(l, seen))
           .filter(Boolean)
           .join(' ') ||
-        (['button', 'submit', 'reset'].includes(e.type) ? e.value : '') ||
+        (e.tagName === 'INPUT' && ['button', 'submit', 'reset'].includes(e.type) ? e.value : '') ||
         e.getAttribute('alt') ||
         (e.tagName === 'INPUT'
           ? ''
@@ -283,6 +283,74 @@ module.exports = function snapshot () {
         }
       }
     }
+    const collected = new Set(actions.map(action => cache.nodes.get(action.node)))
+    const listSelector = 'ul,ol,[role="list"],[role="listbox"],[role="menu"],[role="tablist"]'
+    const pointer = element => {
+      if (getComputedStyle(element).cursor !== 'pointer') return false
+      const parent = flatParent(element)
+      if (!parent || getComputedStyle(parent).cursor !== 'pointer') return true
+      return parent.matches(listSelector)
+    }
+    const inLayer = element => {
+      for (let current = element; current; current = flatParent(current)) {
+        if (current === document.body || current === document.documentElement) return false
+        const position = getComputedStyle(current).position
+        if (position === 'absolute' || position === 'fixed' || position === 'sticky') return true
+      }
+      return false
+    }
+    const pointerLists = new WeakMap()
+    const inPointerList = element => {
+      const parent = flatParent(element)
+      if (!parent?.matches(listSelector)) return false
+      let eligible = pointerLists.get(parent)
+      if (eligible == null) {
+        eligible = [...parent.children].filter(pointer).length >= 2
+        pointerLists.set(parent, eligible)
+      }
+      return eligible
+    }
+    const candidates = []
+    for (const element of queryAll('li,div,span,td,dd,p')) {
+      if (collected.has(element) || !visible(element) || !pointer(element)) continue
+      if (element.matches(':disabled') || cache.closest(element, '[aria-disabled="true"]')) continue
+      if (element.querySelector(selector)) continue
+      const rect = element.getBoundingClientRect()
+      const x = rect.x + rect.width / 2
+      const y = rect.y + rect.height / 2
+      if (
+        rect.width <= 0 ||
+        rect.height <= 0 ||
+        x < 0 ||
+        y < 0 ||
+        x >= innerWidth ||
+        y >= innerHeight
+      ) {
+        continue
+      }
+      const hit = cache.elementFromPoint(x, y)
+      if (hit && !cache.contains(element, hit)) continue
+      if (!inLayer(element) && !inPointerList(element)) continue
+      const label = (name(element) || '').replace(/\s+/g, ' ').trim().slice(0, 120)
+      if (!label) continue
+      candidates.push({ element, rect, label })
+    }
+    const rows = candidates.filter(
+      ({ element }) =>
+        !candidates.some(
+          other => other.element !== element && cache.contains(element, other.element)
+        )
+    )
+    for (const { element, rect, label } of rows.slice(0, 40)) {
+      actions.push({
+        node: identity(element),
+        role: 'button',
+        label,
+        rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
+        kind: 'click',
+        value: ''
+      })
+    }
     const words = []
     const range = document.createRange()
     let length = 0
@@ -354,8 +422,18 @@ module.exports = function snapshot () {
     const height = document.documentElement.scrollHeight
     const pageKey = cache.pageKey()
     const guards = {}
+    const scopes = {}
     for (const a of actions) {
-      if (!(a.node in guards)) guards[a.node] = cache.guard(cache.nodes.get(a.node))
+      if (a.node in guards) continue
+      const element = cache.nodes.get(a.node)
+      guards[a.node] = cache.guard(element)
+      const scope =
+        cache.closest(element, 'form,dialog,[role="dialog"],article,li,tr,[role="row"]') ||
+        flatParent(element)
+      scopes[a.node] =
+        (scope instanceof HTMLElement
+          ? readInnerText.call(scope).slice(0, 1000)
+          : scope?.textContent?.slice(0, 1000)) || ''
     }
     // Compare meaning and identity. Geometry is always resolved and hit-tested just before input.
     const semantics = actions.map(({ rect, ...action }) => action)
@@ -395,6 +473,7 @@ module.exports = function snapshot () {
       marker,
       pageKey,
       guards,
+      scopes,
       omittedActions
     }
   })()