@browserless/agent 14.0.0 → 14.0.1

package/README.md

@@ -219,7 +219,7 @@ provider that did not answer, is in `error` as it was thrown. The reasons:
 | `step_budget` | Action or decision-request budget exhausted |
 | `no_change` | Three consecutive successful actions left the same observation. An action that changed nothing is not offered again until the page changes |
 | `stale_target` | Three consecutive decisions chose the same target and it failed its freshness check each time |
-| `model_blocked` | The provider selected BLOCKED |
+| `model_blocked` | The provider selected BLOCKED, or the text model had no value to type |
 
 Malformed responses, network/provider failures and aborts resolve with
 `status: 'error'` and the original error in `error`; they never become invented
@@ -525,8 +525,9 @@ path and final page it has answered both yes and no.
 
 The text model is called with `generateText` and a JSON object output only for
 TYPE_TEXT. It must return exactly `{ "text": "..." }` with a nonempty string of
-at most 2000 characters. Null, extra keys, arrays, invalid JSON and a reply
-with no output fail closed.
+at most 2000 characters. `{"text": null}` means the goal never gave a value, and the
+run stops with `model_blocked` before anything is typed. Extra keys, arrays,
+invalid JSON and a reply with no output fail closed.
 Reasoning models can spend the whole 1024-token reply budget thinking and
 return no usable value, so reasoning is off unless `reasoning` says otherwise.
 No personal information or missing value is guessed by the executor. Page text,
@@ -535,8 +536,9 @@ only pages you may disclose to them.
 
 ## How it works
 
-1. `page.evaluate` collects visible controls, viewport text, form state and live
-   DOM refs. Persistent DOM node IDs, snapshot action IDs (`e1`, `e2`) and model
+1. Puppeteer's isolated world collects visible controls, viewport text, form
+   state and live DOM refs, so page scripts cannot rewrite the node map.
+   Persistent DOM node IDs, snapshot action IDs (`e1`, `e2`) and model
    indices are separate. At most 250 element actions and 6000 text characters
    are offered per snapshot.
 2. One request proposes an operation plus speculative CLICK, TYPE_TEXT, SUBMIT and

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/agent",
   "description": "Drive a Puppeteer page toward a natural-language goal using finite decisions.",
   "homepage": "https://browserless.js.org",
-  "version": "14.0.0",
+  "version": "14.0.1",
   "main": "src",
   "author": {
     "email": "hello@microlink.io",
@@ -58,5 +58,5 @@
     "access": "public"
   },
   "types": "src/index.d.ts",
-  "gitHead": "bf10b39022a6cc44cccd545756f162bed90dd589"
+  "gitHead": "3a6195884bad55e5b3bd1a9aaf221ceaeb7d3d32"
 }

package/src/browser.js

@@ -21,11 +21,20 @@ const TARGET_GONE = new RegExp(
   `${NAVIGATING.source}|Node is detached from document|not clickable or not an Element`
 )
 
+// Puppeteer's isolated world shares the DOM and hides its window from page scripts.
+const agentWorld = page => {
+  const frame = typeof page.mainFrame === 'function' ? page.mainFrame() : undefined
+  const isolated = typeof frame?.isolatedRealm === 'function' ? frame.isolatedRealm() : undefined
+  return typeof isolated?.evaluate === 'function' ? isolated : page
+}
+
 const snapshotUnlessNavigating = page =>
-  page.evaluate(snapshot).catch(error => {
-    if (NAVIGATING.test(error.message)) return null
-    throw error
-  })
+  agentWorld(page)
+    .evaluate(snapshot)
+    .catch(error => {
+      if (NAVIGATING.test(error.message)) return null
+      throw error
+    })
 
 const observe = async page => {
   const deadline = Date.now() + DOCUMENT_READY_LIMIT_MS
@@ -111,7 +120,7 @@ const settled = (action, limits) =>
   new Promise(resolve => {
     const field = window.__browserlessAgent?.nodes.get(action.node)
     const autocomplete = action.kind === 'fill' && field?.getAttribute('role') === 'combobox'
-    const suggestionVisible = () => {
+    const visibleOptions = () => {
       const ids = (field.getAttribute('aria-controls') || field.getAttribute('aria-owns') || '')
         .split(/\s+/)
         .filter(Boolean)
@@ -120,7 +129,7 @@ const settled = (action, limits) =>
         : [document]
       return roots
         .flatMap(root => [...root.querySelectorAll('[role="option"]')])
-        .some(option => {
+        .filter(option => {
           const r = option.getBoundingClientRect()
           return (
             r.width > 0 &&
@@ -131,6 +140,9 @@ const settled = (action, limits) =>
           )
         })
     }
+    // Options already on screen are not this field's new suggestions.
+    const alreadyVisible = new Set(autocomplete ? visibleOptions() : [])
+    const suggestionVisible = () => visibleOptions().some(option => !alreadyVisible.has(option))
     let frames = 0
     let finished = false
     const finish = () => {
@@ -147,7 +159,10 @@ const settled = (action, limits) =>
   })
 
 // Navigation can destroy the page context mid-wait; the next observation handles that.
-const settle = (page, action) => page.evaluate(settled, action, SETTLE_LIMITS).catch(() => {})
+const settle = (page, action) =>
+  agentWorld(page)
+    .evaluate(settled, action, SETTLE_LIMITS)
+    .catch(() => {})
 
 const execute = async (page, state, action, text, waitMs) => {
   if (action.kind === 'wait') {
@@ -162,7 +177,7 @@ const execute = async (page, state, action, text, waitMs) => {
   }
   let handle
   try {
-    handle = await page.evaluateHandle(
+    handle = await agentWorld(page).evaluateHandle(
       node => window.__browserlessAgent?.nodes.get(node) || null,
       action.node
     )
@@ -200,6 +215,7 @@ const execute = async (page, state, action, text, waitMs) => {
 }
 
 module.exports = {
+  agentWorld,
   readStableOutline,
   pageOutline,
   observe,

package/src/model.js

@@ -18,6 +18,7 @@ const {
   RULES_WRITER,
   GOAL_EVALUATION
 } = require('./questions')
+const { BlockedError } = require('./errors')
 
 const PROBABILITY_SUM_TOLERANCE = 0.02
 const WINNER_TOLERANCE = 1e-6
@@ -33,7 +34,7 @@ const TEXT_HISTORY_LENGTH = 6
 
 const FIELD_VALUE_SCHEMA = jsonSchema({
   type: 'object',
-  properties: { text: { type: 'string' } },
+  properties: { text: { type: ['string', 'null'] } },
   required: ['text'],
   additionalProperties: false
 })
@@ -359,6 +360,9 @@ const fieldText = async (goal, action, state, history, model, options) => {
       'text'
     )
   )
+  if (output?.text === null) {
+    throw new BlockedError('model_blocked', 'The goal does not say what to type in this field.')
+  }
   if (!isFieldValue(output)) throw invalidFieldValue()
   return output.text
 }