@browserless/agent 0.0.0-stage → 14.0.0

package/examples/github.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const agent = require('..')
+const debug = require('debug-logfmt')('browserless:agent')
+const { flat, succeeded } = require('./util')
+
+module.exports = async ({ page, browserless }) => {
+  agent(page)
+
+  await browserless.goto(page, { url: 'https://github.com/microlinkhq/browserless' })
+  const opened = await page.goal('Open the list of open issues of this repository.')
+  debug('open issues', { status: opened.status, ...flat(await opened.profiling()) })
+  succeeded(opened)
+
+  const { data } = succeeded(
+    await page.extract('get the open issues listed on the page', {
+      issues: { attr: { title: { type: 'string' }, url: { type: 'url' } } }
+    })
+  )
+  return data
+}

package/examples/google-flights.js

@@ -0,0 +1,41 @@
+'use strict'
+
+const agent = require('..')
+const debug = require('debug-logfmt')('browserless:agent')
+const { flat, succeeded } = require('./util')
+
+const DAYS_AHEAD = 30
+const DAY_MS = 24 * 60 * 60 * 1000
+const FLIGHT_FIELDS = ['airline', 'departure', 'arrival', 'duration', 'stops', 'price']
+
+const departure = new Date(Date.now() + DAYS_AHEAD * DAY_MS).toLocaleDateString('en-US', {
+  month: 'long',
+  day: 'numeric',
+  year: 'numeric'
+})
+
+module.exports = async ({ page, browserless }) => {
+  agent(page)
+
+  await browserless.goto(page, {
+    url: 'https://www.google.com/travel/flights?hl=en',
+    adblock: false
+  })
+  const consented = await page.goal(
+    'If a cookie consent page is shown, scroll down and reject all cookies.'
+  )
+  debug('reject cookies', { status: consented.status, ...flat(await consented.profiling()) })
+  succeeded(consented)
+  const found = await page.goal(
+    `Find one-way flights from Zurich to London on ${departure}, for one adult in economy. Stop when matching flight options are visible.`
+  )
+  debug('find flights', { status: found.status, ...flat(await found.profiling()) })
+  succeeded(found)
+
+  const { data } = succeeded(
+    await page.extract('get the flights listed on the page', {
+      flights: { attr: Object.fromEntries(FLIGHT_FIELDS.map(name => [name, { type: 'string' }])) }
+    })
+  )
+  return data
+}

package/examples/hacker-news.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const agent = require('..')
+const debug = require('debug-logfmt')('browserless:agent')
+const { flat, succeeded } = require('./util')
+
+module.exports = async ({ page, browserless }) => {
+  agent(page)
+
+  await browserless.goto(page, { url: 'https://news.ycombinator.com' })
+  const opened = await page.goal('Open the comments page of the first story on the front page.')
+  debug('open comments', { status: opened.status, ...flat(await opened.profiling()) })
+  succeeded(opened)
+
+  const { data } = succeeded(
+    await page.extract('get the story title, its points and the comments', {
+      title: { type: 'string' },
+      points: { type: 'number' },
+      comments: { attr: { author: { type: 'string' }, text: { type: 'string' } } }
+    })
+  )
+  return data
+}

package/examples/run.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const agent = require('..')
+const debug = require('debug-logfmt')('browserless:agent')
+const { flat, succeeded } = require('./util')
+
+module.exports = async ({ page, browserless, opts }) => {
+  const { url, goal, extract } = opts
+  if (!url || !goal || !extract) {
+    throw new TypeError(
+      'Usage: browserless exec examples/run.js --url=<url> --goal=<goal> --extract=<what to get>'
+    )
+  }
+  agent(page)
+
+  await browserless.goto(page, { url })
+  const reached = await page.goal(goal)
+  debug('goal', { status: reached.status, ...flat(await reached.profiling()) })
+  succeeded(reached)
+
+  const { data } = succeeded(await page.extract(extract))
+  return data
+}

package/examples/util.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const flat = ({ accuracy, cost, timing, error }) => ({
+  ...accuracy,
+  ...cost,
+  ...timing,
+  ...(error && { error: error.message })
+})
+
+const succeeded = result => {
+  if (result.status === 'error') throw result.error
+  return result
+}
+
+module.exports = { flat, succeeded }

package/examples/wallapop.js

@@ -0,0 +1,26 @@
+'use strict'
+
+const agent = require('..')
+const debug = require('debug-logfmt')('browserless:agent')
+const { flat, succeeded } = require('./util')
+
+module.exports = async ({ page, browserless }) => {
+  agent(page)
+
+  await browserless.goto(page, { url: 'https://wallapop.com' })
+  const searched = await page.goal('busca "bmw x3"')
+  debug('search', { status: searched.status, ...flat(await searched.profiling()) })
+  succeeded(searched)
+  const sorted = await page.goal('ordena los resultados de más barato a más caro')
+  debug('sort', { status: sorted.status, ...flat(await sorted.profiling()) })
+  succeeded(sorted)
+
+  const { data } = succeeded(
+    await page.extract('get the search results', {
+      products: {
+        attr: { name: { type: 'string' }, price: { type: 'number' }, url: { type: 'url' } }
+      }
+    })
+  )
+  return data
+}

package/examples/wikipedia.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const agent = require('..')
+const debug = require('debug-logfmt')('browserless:agent')
+const { flat, succeeded } = require('./util')
+
+module.exports = async ({ page, browserless }) => {
+  agent(page)
+
+  await browserless.goto(page, { url: 'https://en.wikipedia.org/wiki/Main_Page' })
+  const opened = await page.goal(
+    'Search for "Gödel\'s incompleteness theorems" and open that article.'
+  )
+  debug('open article', { status: opened.status, ...flat(await opened.profiling()) })
+  succeeded(opened)
+
+  const { data } = succeeded(
+    await page.extract('get the article title and its first paragraph', {
+      title: { type: 'string' },
+      firstParagraph: { type: 'string' }
+    })
+  )
+  return data
+}

package/package.json

@@ -1,6 +1,62 @@
 {
   "name": "@browserless/agent",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "description": "Drive a Puppeteer page toward a natural-language goal using finite decisions.",
+  "homepage": "https://browserless.js.org",
+  "version": "14.0.0",
+  "main": "src",
+  "author": {
+    "email": "hello@microlink.io",
+    "name": "microlink.io",
+    "url": "https://microlink.io"
+  },
+  "repository": {
+    "directory": "packages/agent",
+    "type": "git",
+    "url": "git+https://github.com/microlinkhq/browserless.git#master"
+  },
+  "bugs": {
+    "url": "https://github.com/microlinkhq/browserless/issues"
+  },
+  "keywords": [
+    "agent",
+    "ai-sdk",
+    "browserless",
+    "jev",
+    "puppeteer"
+  ],
+  "dependencies": {
+    "ai": "7",
+    "debug-logfmt": "~1.4.15"
+  },
+  "devDependencies": {
+    "@ai-sdk/typesafe-ai": "~3.0.13",
+    "@browserless/cli": "14.0.0",
+    "ava": "5",
+    "jsdom": "26",
+    "puppeteer": "25"
+  },
+  "engines": {
+    "node": ">= 24"
+  },
+  "files": [
+    "examples",
+    "src",
+    "LICENSE.md"
+  ],
+  "scripts": {
+    "test": "ava",
+    "start": "browserless exec examples/wallapop.js --no-headless",
+    "benchmark": "browserless exec scripts/benchmark.js",
+    "compare": "browserless exec scripts/compare.js"
+  },
+  "license": "MIT",
+  "ava": {
+    "timeout": "30s",
+    "workerThreads": false
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "types": "src/index.d.ts",
+  "gitHead": "bf10b39022a6cc44cccd545756f162bed90dd589"
+}

package/src/browser.js

@@ -0,0 +1,212 @@
+/* global location, innerWidth, innerHeight, getSelection, requestAnimationFrame */
+'use strict'
+
+const snapshot = require('./snapshot')
+const pageOutline = require('./outline')
+const { BlockedError, StaleDecisionError } = require('./errors')
+
+const KEYBOARD_TARGET = { focused: true }
+const SETTLE_LIMITS = { autocompleteMs: 200, defaultMs: 50, minFrames: 2 }
+const DOCUMENT_READY_LIMIT_MS = 3000
+const DOCUMENT_READY_POLL_MS = 50
+const OUTLINE_LIMITS = { characters: 60000, text: 80, attribute: 80, siblings: 3 }
+const OUTLINE_STABLE_POLL_MS = 250
+const OUTLINE_STABLE_LIMIT_MS = 3000
+
+const pageChanged = (before, after) =>
+  JSON.stringify(before.marker) !== JSON.stringify(after.marker)
+
+const NAVIGATING = /Execution context was destroyed|Cannot find context with specified id/
+const TARGET_GONE = new RegExp(
+  `${NAVIGATING.source}|Node is detached from document|not clickable or not an Element`
+)
+
+const snapshotUnlessNavigating = page =>
+  page.evaluate(snapshot).catch(error => {
+    if (NAVIGATING.test(error.message)) return null
+    throw error
+  })
+
+const observe = async page => {
+  const deadline = Date.now() + DOCUMENT_READY_LIMIT_MS
+  while (true) {
+    const state = await snapshotUnlessNavigating(page)
+    if (state) return state
+    if (Date.now() >= deadline) {
+      throw new BlockedError('unsupported_surface', 'No document body is available.')
+    }
+    await new Promise(resolve => setTimeout(resolve, DOCUMENT_READY_POLL_MS))
+  }
+}
+
+// Geometry is resolved now, not from model-time coordinates. Form state is local
+// to the target; unrelated page changes do not invalidate this decision.
+const targetFresh = (element, state, action, requirements = {}) => {
+  const cache = window.__browserlessAgent
+  if (
+    !cache ||
+    !element?.isConnected ||
+    cache.nodes.get(action.node) !== element ||
+    JSON.stringify(cache.guard(element)) !== JSON.stringify(state.guards[action.node]) ||
+    performance.timeOrigin !== state.pageKey[0] ||
+    location.href !== state.url ||
+    cache.closest(element, '[aria-disabled="true"],[inert]') ||
+    ['password', 'file', 'hidden'].includes(element.type)
+  ) {
+    return false
+  }
+  if (requirements.focused && cache.activeElement() !== element) return false
+  const r = element.getBoundingClientRect()
+  const x = r.x + r.width / 2
+  const y = r.y + r.height / 2
+  if (
+    !r.width ||
+    !r.height ||
+    x < 0 ||
+    y < 0 ||
+    x >= innerWidth ||
+    y >= innerHeight ||
+    !cache.contains(element, cache.elementFromPoint(x, y))
+  ) {
+    return false
+  }
+  if (
+    action.kind === 'fill' &&
+    (element.readOnly || element.getAttribute('aria-readonly') === 'true')
+  ) {
+    return false
+  }
+  if (
+    action.kind === 'select' &&
+    (element.tagName !== 'SELECT' ||
+      ![...element.options].some(
+        o => o.value === action.value && !o.disabled && !o.closest('optgroup[disabled]')
+      ))
+  ) {
+    return false
+  }
+  return true
+}
+
+const selectContents = element => {
+  if (typeof element.select === 'function') element.select()
+  else getSelection().selectAllChildren(element)
+}
+
+const readOutline = page => page.evaluate(pageOutline, OUTLINE_LIMITS)
+
+const readStableOutline = async page => {
+  const deadline = Date.now() + OUTLINE_STABLE_LIMIT_MS
+  let current = await readOutline(page)
+  while (Date.now() < deadline) {
+    await new Promise(resolve => setTimeout(resolve, OUTLINE_STABLE_POLL_MS))
+    const next = await readOutline(page)
+    if (next.outline === current.outline) return next
+    current = next
+  }
+  return current
+}
+
+const settled = (action, limits) =>
+  new Promise(resolve => {
+    const field = window.__browserlessAgent?.nodes.get(action.node)
+    const autocomplete = action.kind === 'fill' && field?.getAttribute('role') === 'combobox'
+    const suggestionVisible = () => {
+      const ids = (field.getAttribute('aria-controls') || field.getAttribute('aria-owns') || '')
+        .split(/\s+/)
+        .filter(Boolean)
+      const roots = ids.length
+        ? ids.map(id => field.getRootNode().getElementById(id)).filter(Boolean)
+        : [document]
+      return roots
+        .flatMap(root => [...root.querySelectorAll('[role="option"]')])
+        .some(option => {
+          const r = option.getBoundingClientRect()
+          return (
+            r.width > 0 &&
+            r.height > 0 &&
+            r.bottom > 0 &&
+            r.top < innerHeight &&
+            option.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true })
+          )
+        })
+    }
+    let frames = 0
+    let finished = false
+    const finish = () => {
+      finished = true
+      resolve()
+    }
+    const onFrame = () => {
+      if (finished) return
+      if (++frames >= limits.minFrames && (!autocomplete || suggestionVisible())) finish()
+      else requestAnimationFrame(onFrame)
+    }
+    setTimeout(finish, autocomplete ? limits.autocompleteMs : limits.defaultMs)
+    requestAnimationFrame(onFrame)
+  })
+
+// Navigation can destroy the page context mid-wait; the next observation handles that.
+const settle = (page, action) => page.evaluate(settled, action, SETTLE_LIMITS).catch(() => {})
+
+const execute = async (page, state, action, text, waitMs) => {
+  if (action.kind === 'wait') {
+    await new Promise(resolve => setTimeout(resolve, waitMs))
+    return
+  }
+  if (action.kind === 'scroll') {
+    if (pageChanged(state, await observe(page))) throw new StaleDecisionError()
+    await page.mouse.move(state.w / 2, state.h / 2)
+    await page.mouse.wheel({ deltaY: action.delta })
+    return
+  }
+  let handle
+  try {
+    handle = await page.evaluateHandle(
+      node => window.__browserlessAgent?.nodes.get(node) || null,
+      action.node
+    )
+    const element = handle.asElement()
+    const assertFresh = async (requirements = {}) => {
+      if (!(await element?.evaluate(targetFresh, state, action, requirements))) {
+        throw new StaleDecisionError()
+      }
+    }
+    // Focus can trigger menus/re-renders, so freshness is checked again after it.
+    const focusForKeyboard = async () => {
+      await element.focus()
+      await assertFresh(KEYBOARD_TARGET)
+    }
+    await assertFresh()
+    if (action.kind === 'fill') {
+      await focusForKeyboard()
+      await element.evaluate(selectContents)
+      await assertFresh(KEYBOARD_TARGET)
+      await page.keyboard.sendCharacter(text)
+    } else if (action.kind === 'submit') {
+      await focusForKeyboard()
+      await page.keyboard.press('Enter')
+    } else if (action.kind === 'select') {
+      await element.select(action.value)
+    } else if (action.kind === 'click') {
+      await element.click()
+    } else throw new TypeError('Unknown observed action.')
+  } catch (error) {
+    if (TARGET_GONE.test(error.message)) throw new StaleDecisionError()
+    throw error
+  } finally {
+    await handle?.dispose().catch(() => {})
+  }
+}
+
+module.exports = {
+  readStableOutline,
+  pageOutline,
+  observe,
+  execute,
+  settle,
+  settled,
+  pageChanged,
+  targetFresh,
+  selectContents
+}

package/src/errors.js

@@ -0,0 +1,20 @@
+'use strict'
+
+class BlockedError extends Error {
+  constructor (reason, message, trace = []) {
+    super(message)
+    this.name = 'BlockedError'
+    this.code = 'BLOCKED'
+    this.reason = reason
+    this.trace = trace
+  }
+}
+
+class StaleDecisionError extends Error {
+  constructor () {
+    super('Target or nearby context changed; observe again before input.')
+    this.name = 'StaleDecisionError'
+  }
+}
+
+module.exports = { BlockedError, StaleDecisionError }

package/src/index.d.ts

@@ -0,0 +1,108 @@
+import type { Page } from 'puppeteer' with { 'resolution-mode': 'import' }
+import type { Experimental_DecisionModel, LanguageModel } from 'ai' with { 'resolution-mode': 'import' }
+
+declare function agent<PAGE extends Page>(page: PAGE, defaults?: agent.Options): PAGE & agent.Methods
+
+declare namespace agent {
+  type BlockedReason = 'captcha' | 'login_wall' | 'unsupported_surface' | 'step_budget' | 'no_change' | 'stale_target' | 'model_blocked'
+  type Reasoning = 'provider-default' | 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'
+  interface Rule {
+    selector?: string | string[]
+    selectorAll?: string | string[]
+    attr?: string | string[] | Rules
+    type?: string
+  }
+  type Rules = Record<string, Rule | Rule[]>
+  type Data = Record<string, unknown>
+  type Extractor = (page: Page, rules: Rules) => Promise<Data>
+  interface Methods {
+    goal(goal: string, options?: Options): Promise<Result>
+    extract(rules: Rules, options?: Options): Promise<Extracted>
+    extract(instruction: string, rules?: Rules, options?: Options): Promise<Extracted>
+  }
+  function goal(page: Page, goal: string, options?: Options): Promise<Result>
+  function extract(page: Page, rules: Rules, options?: Options): Promise<Extracted>
+  function extract(page: Page, instruction: string, rules?: Rules, options?: Options): Promise<Extracted>
+  const applyRules: Extractor
+  interface Options {
+    extractor?: Extractor
+    decisions?: Experimental_DecisionModel | false
+    text?: LanguageModel
+    evaluator?: Experimental_DecisionModel
+    reasoning?: Reasoning
+    maxSteps?: number
+    maxDecisions?: number
+    waitMs?: number
+    timeout?: number
+    signal?: AbortSignal
+  }
+  interface TraceEntry {
+    operation: 'CLICK' | 'TYPE_TEXT' | 'SELECT' | 'SUBMIT' | 'SCROLL_UP' | 'SCROLL_DOWN' | 'WAIT' | 'DONE' | 'BLOCKED'
+    action?: string
+    confidence?: number
+    probabilities?: Record<string, number>
+    decisionMs: number
+    textMs?: number
+    targetConfidence?: number
+    targetProbabilities?: Record<string, number>
+    step: number
+    decision: number
+    text?: string
+    stale?: boolean
+    pageChanged?: boolean
+  }
+  interface Accuracy {
+    passed: boolean
+    probability: number
+  }
+  interface Cost {
+    calls: number
+    inputTokens: number | undefined
+    outputTokens: number | undefined
+    cachedInputTokens: number
+    cacheWriteTokens: number
+    reasoningTokens: number
+    usd: number | undefined
+    generationIds: string[]
+  }
+  interface Timing {
+    totalMs: number
+    modelMs: number
+    otherMs: number
+    evaluationMs: number
+  }
+  interface Profiling {
+    accuracy: Accuracy | undefined
+    cost: Cost
+    timing: Timing
+    error?: Error
+  }
+  interface ProfilingOptions {
+    signal?: AbortSignal
+  }
+  type ProfilingFunction = (options?: ProfilingOptions) => Promise<Profiling>
+  interface Run {
+    steps: number
+    decisions: number
+    trace: TraceEntry[]
+    profiling: ProfilingFunction
+  }
+  type Result = (Run & { status: 'success' }) | (Run & { status: 'error'; error: Error })
+  interface ExtractProfiling {
+    rules: Rules | undefined
+    cost: Cost
+    timing: Timing
+  }
+  type ExtractProfilingFunction = () => Promise<ExtractProfiling>
+  type Extracted =
+    | { status: 'success'; data: Data; profiling: ExtractProfilingFunction }
+    | { status: 'error'; error: Error; profiling: ExtractProfilingFunction }
+  class BlockedError extends Error {
+    constructor(reason: BlockedReason, message: string, trace?: TraceEntry[])
+    code: 'BLOCKED'
+    reason: BlockedReason
+    trace: TraceEntry[]
+  }
+}
+
+export = agent