@browserless/agent 14.0.0 → 14.0.2

package/README.md

@@ -219,7 +219,7 @@ provider that did not answer, is in `error` as it was thrown. The reasons:
 | `step_budget` | Action or decision-request budget exhausted |
 | `no_change` | Three consecutive successful actions left the same observation. An action that changed nothing is not offered again until the page changes |
 | `stale_target` | Three consecutive decisions chose the same target and it failed its freshness check each time |
-| `model_blocked` | The provider selected BLOCKED |
+| `model_blocked` | The provider selected BLOCKED, or the text model had no value to type |
 
 Malformed responses, network/provider failures and aborts resolve with
 `status: 'error'` and the original error in `error`; they never become invented
@@ -525,8 +525,9 @@ path and final page it has answered both yes and no.
 
 The text model is called with `generateText` and a JSON object output only for
 TYPE_TEXT. It must return exactly `{ "text": "..." }` with a nonempty string of
-at most 2000 characters. Null, extra keys, arrays, invalid JSON and a reply
-with no output fail closed.
+at most 2000 characters. `{"text": null}` means the goal never gave a value, and the
+run stops with `model_blocked` before anything is typed. Extra keys, arrays,
+invalid JSON and a reply with no output fail closed.
 Reasoning models can spend the whole 1024-token reply budget thinking and
 return no usable value, so reasoning is off unless `reasoning` says otherwise.
 No personal information or missing value is guessed by the executor. Page text,
@@ -535,8 +536,9 @@ only pages you may disclose to them.
 
 ## How it works
 
-1. `page.evaluate` collects visible controls, viewport text, form state and live
-   DOM refs. Persistent DOM node IDs, snapshot action IDs (`e1`, `e2`) and model
+1. Puppeteer's isolated world collects visible controls, viewport text, form
+   state and live DOM refs, so page scripts cannot rewrite the node map.
+   Persistent DOM node IDs, snapshot action IDs (`e1`, `e2`) and model
    indices are separate. At most 250 element actions and 6000 text characters
    are offered per snapshot.
 2. One request proposes an operation plus speculative CLICK, TYPE_TEXT, SUBMIT and

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/agent",
   "description": "Drive a Puppeteer page toward a natural-language goal using finite decisions.",
   "homepage": "https://browserless.js.org",
-  "version": "14.0.0",
+  "version": "14.0.2",
   "main": "src",
   "author": {
     "email": "hello@microlink.io",
@@ -47,6 +47,7 @@
     "test": "ava",
     "start": "browserless exec examples/wallapop.js --no-headless",
     "benchmark": "browserless exec scripts/benchmark.js",
+    "perf": "node scripts/perf-baseline.js",
     "compare": "browserless exec scripts/compare.js"
   },
   "license": "MIT",
@@ -58,5 +59,5 @@
     "access": "public"
   },
   "types": "src/index.d.ts",
-  "gitHead": "bf10b39022a6cc44cccd545756f162bed90dd589"
+  "gitHead": "2f8b09a1509de4868c5ca2981ed147af8638968d"
 }

package/src/browser.js

@@ -21,11 +21,20 @@ const TARGET_GONE = new RegExp(
   `${NAVIGATING.source}|Node is detached from document|not clickable or not an Element`
 )
 
+// Puppeteer's isolated world shares the DOM and hides its window from page scripts.
+const agentWorld = page => {
+  const frame = typeof page.mainFrame === 'function' ? page.mainFrame() : undefined
+  const isolated = typeof frame?.isolatedRealm === 'function' ? frame.isolatedRealm() : undefined
+  return typeof isolated?.evaluate === 'function' ? isolated : page
+}
+
 const snapshotUnlessNavigating = page =>
-  page.evaluate(snapshot).catch(error => {
-    if (NAVIGATING.test(error.message)) return null
-    throw error
-  })
+  agentWorld(page)
+    .evaluate(snapshot)
+    .catch(error => {
+      if (NAVIGATING.test(error.message)) return null
+      throw error
+    })
 
 const observe = async page => {
   const deadline = Date.now() + DOCUMENT_READY_LIMIT_MS
@@ -111,7 +120,7 @@ const settled = (action, limits) =>
   new Promise(resolve => {
     const field = window.__browserlessAgent?.nodes.get(action.node)
     const autocomplete = action.kind === 'fill' && field?.getAttribute('role') === 'combobox'
-    const suggestionVisible = () => {
+    const visibleOptions = () => {
       const ids = (field.getAttribute('aria-controls') || field.getAttribute('aria-owns') || '')
         .split(/\s+/)
         .filter(Boolean)
@@ -120,7 +129,7 @@ const settled = (action, limits) =>
         : [document]
       return roots
         .flatMap(root => [...root.querySelectorAll('[role="option"]')])
-        .some(option => {
+        .filter(option => {
           const r = option.getBoundingClientRect()
           return (
             r.width > 0 &&
@@ -131,6 +140,9 @@ const settled = (action, limits) =>
           )
         })
     }
+    // Options already on screen are not this field's new suggestions.
+    const alreadyVisible = new Set(autocomplete ? visibleOptions() : [])
+    const suggestionVisible = () => visibleOptions().some(option => !alreadyVisible.has(option))
     let frames = 0
     let finished = false
     const finish = () => {
@@ -147,7 +159,10 @@ const settled = (action, limits) =>
   })
 
 // Navigation can destroy the page context mid-wait; the next observation handles that.
-const settle = (page, action) => page.evaluate(settled, action, SETTLE_LIMITS).catch(() => {})
+const settle = (page, action) =>
+  agentWorld(page)
+    .evaluate(settled, action, SETTLE_LIMITS)
+    .catch(() => {})
 
 const execute = async (page, state, action, text, waitMs) => {
   if (action.kind === 'wait') {
@@ -162,7 +177,7 @@ const execute = async (page, state, action, text, waitMs) => {
   }
   let handle
   try {
-    handle = await page.evaluateHandle(
+    handle = await agentWorld(page).evaluateHandle(
       node => window.__browserlessAgent?.nodes.get(node) || null,
       action.node
     )
@@ -200,6 +215,7 @@ const execute = async (page, state, action, text, waitMs) => {
 }
 
 module.exports = {
+  agentWorld,
   readStableOutline,
   pageOutline,
   observe,

package/src/model.js

@@ -18,6 +18,7 @@ const {
   RULES_WRITER,
   GOAL_EVALUATION
 } = require('./questions')
+const { BlockedError } = require('./errors')
 
 const PROBABILITY_SUM_TOLERANCE = 0.02
 const WINNER_TOLERANCE = 1e-6
@@ -33,7 +34,7 @@ const TEXT_HISTORY_LENGTH = 6
 
 const FIELD_VALUE_SCHEMA = jsonSchema({
   type: 'object',
-  properties: { text: { type: 'string' } },
+  properties: { text: { type: ['string', 'null'] } },
   required: ['text'],
   additionalProperties: false
 })
@@ -359,6 +360,9 @@ const fieldText = async (goal, action, state, history, model, options) => {
       'text'
     )
   )
+  if (output?.text === null) {
+    throw new BlockedError('model_blocked', 'The goal does not say what to type in this field.')
+  }
   if (!isFieldValue(output)) throw invalidFieldValue()
   return output.text
 }

package/src/snapshot.js

@@ -1,4 +1,4 @@
-/* global location, scrollX, scrollY, innerWidth, innerHeight, NodeFilter */
+/* global location, scrollX, scrollY, innerWidth, innerHeight */
 // Adapted from browser-use/jev-ultrafast snapshot.js (MIT). See README.md.
 module.exports = function snapshot () {
   return (() => {
@@ -54,9 +54,39 @@ module.exports = function snapshot () {
       return active
     }
     const safe = e => !['password', 'file', 'hidden'].includes(e.type)
-    const visible = e =>
-      !cache.closest(e, '[aria-hidden="true"],[inert]') &&
-      e.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true })
+    const hiddenMemo = new WeakMap()
+    const visibleMemo = new WeakMap()
+    const hiddenAncestor = element => {
+      const seen = []
+      for (let current = element; current; current = flatParent(current)) {
+        if (hiddenMemo.has(current)) {
+          const found = hiddenMemo.get(current)
+          for (const item of seen) hiddenMemo.set(item, found)
+          return found
+        }
+        seen.push(current)
+        if (current.matches('[aria-hidden="true"],[inert]')) {
+          for (const item of seen) hiddenMemo.set(item, current)
+          return current
+        }
+      }
+      for (const item of seen) hiddenMemo.set(item, null)
+      return null
+    }
+    const visible = e => {
+      if (visibleMemo.has(e)) return visibleMemo.get(e)
+      const shown =
+        !hiddenAncestor(e) && e.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true })
+      visibleMemo.set(e, shown)
+      return shown
+    }
+    // cache.guard runs again at input time, so it must not reuse this snapshot's memos.
+    const visibleNow = e => {
+      for (let current = e; current; current = flatParent(current)) {
+        if (current.matches('[aria-hidden="true"],[inert]')) return false
+      }
+      return e.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true })
+    }
     const name = (e, seen = new Set()) => {
       if (!e || seen.has(e)) return ''
       seen.add(e)
@@ -140,7 +170,7 @@ module.exports = function snapshot () {
         .map(e => [identity(e), e.value, e.checked, e.selectedIndex, e.disabled, e.readOnly])
     ]
     cache.guard = e => {
-      if (!e?.isConnected || !visible(e)) return null
+      if (!e?.isConnected || !visibleNow(e)) return null
       const scope =
         cache.closest(e, 'form,dialog,[role="dialog"],article,li,tr,[role="row"]') || flatParent(e)
       return [
@@ -252,37 +282,65 @@ module.exports = function snapshot () {
     const words = []
     const range = document.createRange()
     let length = 0
-    for (const root of roots) {
-      const walker = document.createTreeWalker(
-        root === document ? document.body : root,
-        NodeFilter.SHOW_TEXT
-      )
-      let node
-      while ((node = walker.nextNode()) && length < 6000) {
-        const value = node.textContent.trim()
-        const parent = node.parentElement || root.host
-        if (
-          !value ||
-          !parent ||
-          parent.closest('script,style,noscript,template') ||
-          !visible(parent)
-        ) {
-          continue
-        }
+    const TEXT_SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE'])
+    const intersectsViewport = rect =>
+      rect.width > 0 &&
+      rect.height > 0 &&
+      rect.bottom > 0 &&
+      rect.top < innerHeight &&
+      rect.right > 0 &&
+      rect.left < innerWidth
+    const outsideViewport = rect =>
+      rect.width > 0 &&
+      rect.height > 0 &&
+      (rect.bottom <= 0 || rect.top >= innerHeight || rect.right <= 0 || rect.left >= innerWidth)
+    const fullyInside = rect =>
+      rect.top >= 0 &&
+      rect.left >= 0 &&
+      rect.bottom <= innerHeight &&
+      rect.right <= innerWidth &&
+      rect.width > 0 &&
+      rect.height > 0
+    const pushText = (node, skipRect) => {
+      if (length >= 6000 || node.nodeType !== 3) return
+      const value = node.textContent.trim()
+      if (!value) return
+      if (!skipRect) {
         range.selectNodeContents(node)
-        const r = range.getBoundingClientRect()
-        if (
-          r.width > 0 &&
-          r.height > 0 &&
-          r.bottom > 0 &&
-          r.top < innerHeight &&
-          r.right > 0 &&
-          r.left < innerWidth
-        ) {
-          words.push(value)
-          length += value.length
+        if (!intersectsViewport(range.getBoundingClientRect())) return
+      }
+      words.push(value)
+      length += value.length
+    }
+    // Off-screen subtrees are skipped. A fixed node inside one of those subtrees is not read.
+    const collectText = element => {
+      if (length >= 6000 || TEXT_SKIP.has(element.tagName)) return
+      const rect = element.getBoundingClientRect()
+      if (outsideViewport(rect)) return
+      const inside = fullyInside(rect)
+      if (!intersectsViewport(rect) || !visible(element)) {
+        for (const node of element.childNodes) {
+          if (node.nodeType === 1) collectText(node)
         }
+        return
+      }
+      for (const node of element.childNodes) {
+        if (length >= 6000) return
+        if (node.nodeType === 3) pushText(node, inside)
+        else if (node.nodeType === 1) collectText(node)
+      }
+    }
+    for (const root of roots) {
+      if (length >= 6000) break
+      if (root === document) {
+        if (document.body) collectText(document.body)
+        continue
+      }
+      const host = root.host
+      if (host && visible(host)) {
+        for (const node of root.childNodes) pushText(node)
       }
+      for (const child of root.children) collectText(child)
     }
     const unsupported = []
     for (const e of queryAll('iframe,canvas,input[type="password"],input[type="file"]')) {