@frontera-sdk/cli 1.52.39 → 1.52.40

package/src/mcp/browser/headless-session.ts

@@ -0,0 +1,413 @@
+import type { BrowserContext } from 'playwright-core'
+
+import { acquireChromium } from '../headless'
+import { parseKey } from './keys'
+import { DEVICE_SIZES, HeadlessTab, isProjectAddress, networkText, siteOf, type DeviceWidth } from './headless-tab'
+import { parseWaitInput, WAIT_MAX_MS, type Wait } from './wait'
+
+/**
+ * The `browser_*` tools without Forge: a headless browser this MCP server
+ * process owns. One Chromium context for the process (so a Studio sign-in
+ * holds across tabs), tabs that belong to the session that opened them, and
+ * the answers Forge's Browser gives, word for word where it can: a snapshot
+ * with refs that keep naming their element until the page goes to another
+ * document, waits, every action, console, network and screenshots.
+ *
+ * What differs is that nobody watches. Reading and navigating are free on
+ * any site, as in Forge. What Forge asks the person about before doing it
+ * outside the project (a click that is not a link, submitting a form, Enter
+ * or Space, running script, OK on a dialog, on an address that is not this
+ * machine's) is refused here in words: a headless browser has nobody to ask.
+ * Never, anywhere: typing into a password field.
+ */
+
+type Content = { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string }
+export interface HeadlessAnswer {
+  content: Content[]
+  isError?: true
+  /** Where a screenshot belongs, as Forge says it: the tab, its address and its project's folder. */
+  meta?: { tabId: string; url: string; root: string }
+}
+
+/** `browser_evaluate`'s answer, at most this many characters of JSON. */
+export const EVALUATE_CAP = 20_000
+/** The tabs close, and Chromium with them, after this long without a call. */
+export const SESSION_IDLE_MS = 15 * 60_000
+/** At most this many tabs: the oldest closes when one more opens. */
+const MAX_TABS = 12
+
+const text = (t: string): HeadlessAnswer => ({ content: [{ type: 'text', text: t }] })
+const refuse = (t: string): HeadlessAnswer => ({ isError: true, content: [{ type: 'text', text: t }] })
+const str = (v: unknown): string | undefined => (typeof v === 'string' && v.trim() ? v.trim() : undefined)
+const seconds = (ms: number) => `${(ms / 1000).toFixed(1)} s`
+
+const NOBODY_WATCHES = 'needs the person\'s yes, and nobody watches a headless browser'
+
+function dialogNote(tab: HeadlessTab): string {
+  const d = tab.dialog!
+  const kind = d.type === 'beforeunload' ? 'leave-page' : d.type
+  return `The page opened a ${kind} dialog${d.message ? `: ${JSON.stringify(d.message)}` : ''}${d.type === 'prompt' && d.defaultPrompt ? ` (default ${JSON.stringify(d.defaultPrompt)})` : ''}. The page waits until it is answered: call browser_handle_dialog with accept true or false.`
+}
+
+/** Tools that read, or answer the dialog: everything else waits for the page, which a dialog holds. */
+const DIALOG_SAFE = new Set(['browser_handle_dialog', 'browser_console', 'browser_network', 'browser_screenshot', 'browser_tabs'])
+
+export class HeadlessBrowserSession {
+  private tabs = new Map<string, HeadlessTab>()
+  /** Per project folder: the tab the session chose with browser_tabs select. */
+  private chosen = new Map<string, string>()
+  private context: Promise<{ context: BrowserContext; release: () => void }> | null = null
+  private idle: ReturnType<typeof setTimeout> | null = null
+
+  constructor(private readonly idleMs = SESSION_IDLE_MS) {}
+
+  private touch(): void {
+    if (this.idle) clearTimeout(this.idle)
+    this.idle = setTimeout(() => void this.close(), this.idleMs)
+    this.idle.unref?.()
+  }
+
+  private async browserContext(): Promise<BrowserContext> {
+    if (!this.context) {
+      const opening = (async () => {
+        const { browser, release } = await acquireChromium()
+        try {
+          const context = await browser.newContext({ viewport: { width: DEVICE_SIZES.desktop.width, height: DEVICE_SIZES.desktop.height } })
+          context.on('close', () => {
+            if (this.context === opening) {
+              this.context = null
+              this.tabs.clear()
+              release()
+            }
+          })
+          return { context, release }
+        } catch (err) {
+          release()
+          throw err
+        }
+      })()
+      this.context = opening
+      opening.catch(() => {
+        if (this.context === opening) this.context = null
+      })
+    }
+    return (await this.context).context
+  }
+
+  /** Close every tab and give Chromium back. */
+  async close(): Promise<void> {
+    if (this.idle) clearTimeout(this.idle)
+    this.idle = null
+    const opening = this.context
+    this.context = null
+    this.tabs.clear()
+    this.chosen.clear()
+    if (!opening) return
+    const held = await opening.catch(() => null)
+    if (!held) return
+    await held.context.close().catch(() => {})
+    held.release()
+  }
+
+  private async open(root: string): Promise<HeadlessTab> {
+    const context = await this.browserContext()
+    const mine = [...this.tabs.values()].filter((t) => t.alive)
+    if (mine.length >= MAX_TABS) {
+      const oldest = mine.sort((a, b) => a.agentAt - b.agentAt)[0]!
+      this.tabs.delete(oldest.id)
+      await oldest.close()
+    }
+    const tab = await HeadlessTab.open(await context.newPage(), root)
+    this.tabs.set(tab.id, tab)
+    tab.page.on('close', () => this.tabs.delete(tab.id))
+    return tab
+  }
+
+  /** The tab named, the one chosen, or the one acted in last, of this project. */
+  private tabFor(root: string, input: Record<string, unknown>): HeadlessTab | HeadlessAnswer {
+    const id = str(input.tabId)
+    if (id) {
+      const tab = this.tabs.get(id)
+      return tab && tab.alive && tab.root === root ? tab : refuse(`No tab "${id}" in this project; browser_tabs lists them.`)
+    }
+    const pick = this.chosen.get(root)
+    const picked = pick ? this.tabs.get(pick) : undefined
+    if (picked?.alive) return picked
+    const mine = [...this.tabs.values()].filter((t) => t.alive && t.root === root).sort((a, b) => b.agentAt - a.agentAt)
+    return mine[0] ?? refuse('You have no tab open: browser_open opens one.')
+  }
+
+  private async snapshotText(tab: HeadlessTab, lead: () => string, wait: Wait | null = null): Promise<HeadlessAnswer> {
+    await tab.settle(4_000, 350)
+    let waited = ''
+    let met = true
+    if (wait) {
+      const r = await tab.waitFor(wait.condition, wait.timeoutMs)
+      met = r.met
+      waited = r.error ? `\n${r.error}` : r.met ? `\nWaited ${seconds(r.waitedMs)} for ${wait.what}.` : `\nWaited ${seconds(r.waitedMs)} for ${wait.what}; it did not happen. This is the page now.`
+    }
+    if (tab.dialog) return text(`${lead()}${waited}\n\n${dialogNote(tab)}`)
+    const snap = await tab.takeSnapshot()
+    const answer = `${lead()}${waited}\n\n${snap.text}`
+    return met ? text(answer) : refuse(answer)
+  }
+
+  private async nodeFor(tab: HeadlessTab, ref: string | undefined): Promise<number | HeadlessAnswer> {
+    if (!ref) return refuse('ref is required: take a browser_snapshot and use an element\'s [ref=…].')
+    if (!tab.snapshot) return refuse('Take a browser_snapshot first: refs come from it.')
+    if (tab.navigatedAt > tab.snapshotAt) {
+      const snap = await tab.takeSnapshot()
+      return refuse(`The page went to another document after your snapshot, so look again before acting. Here is the page now; act on these refs.\n\n${snap.text}`)
+    }
+    return tab.nodeForRef(ref) ?? refuse(`No element [ref=${ref}] on this page: take a new browser_snapshot.`)
+  }
+
+  /**
+   * Acting outside the project needs a yes nobody can give here: the
+   * refusal, or null to go ahead. An element in a frame (another site
+   * embedded in the page) is acted on in that frame's site.
+   */
+  private permitted(tab: HeadlessTab, action: string, node?: number): HeadlessAnswer | null {
+    const frame = node !== undefined ? tab.frameAddressOf(node) : null
+    const url = [tab.url, frame].find((u): u is string => !!u && !isProjectAddress(u))
+    if (!url) return null
+    return refuse(`To ${action} on ${siteOf(url)} ${NOBODY_WATCHES}. Reading and navigating there are fine. Open this project in Forge to do it with the person watching, or ask them in the terminal.`)
+  }
+
+  private elementName(tab: HeadlessTab, input: Record<string, unknown>, fallback: string): string {
+    const said = str(input.element)
+    if (said) return said
+    const ref = str(input.ref)
+    const label = ref ? tab.snapshot?.labels.get(ref) : undefined
+    return label ? `${label.role} "${label.name}"` : fallback
+  }
+
+  /** Run one `browser_*` tool for the project at `root`. */
+  async run(tool: string, input: Record<string, unknown>, root: string): Promise<HeadlessAnswer> {
+    this.touch()
+    try {
+      return await this.dispatch(tool, input, root)
+    } finally {
+      this.touch()
+    }
+  }
+
+  private async dispatch(tool: string, input: Record<string, unknown>, root: string): Promise<HeadlessAnswer> {
+    if (tool === 'browser_open') {
+      const url = str(input.url)
+      if (!url) return refuse('url is required.')
+      const wait = parseWaitInput(input.waitFor as Record<string, unknown> | undefined, null)
+      if (wait && 'error' in wait) return refuse(wait.error)
+      const tab = await this.open(root)
+      tab.addSecret(str(input.redact))
+      tab.noteSecretsIn(url)
+      const shown = tab.redact(url)
+      const error = await tab.load(url)
+      if (error) return refuse(`${shown} did not load: ${error}`)
+      return this.snapshotText(tab, () => `Opened ${tab.url && tab.url !== 'about:blank' ? tab.url : shown} in tab ${tab.id}.`, wait)
+    }
+
+    if (tool === 'browser_tabs') {
+      const action = str(input.action) ?? 'list'
+      if (action === 'list') {
+        const current = this.tabFor(root, {})
+        const tabs = await Promise.all(
+          [...this.tabs.values()]
+            .filter((t) => t.alive && t.root === root)
+            .map(async (t) => ({ tabId: t.id, title: await t.title(), url: t.url, owner: 'you', active: current instanceof HeadlessTab && current.id === t.id, ...(t.dialog ? { dialog: true } : {}) })),
+        )
+        return text(JSON.stringify({ tabs }, null, 2))
+      }
+      const tab = this.tabFor(root, input)
+      if (!(tab instanceof HeadlessTab)) return tab
+      if (action === 'select') {
+        this.chosen.set(root, tab.id)
+        tab.agentAt = Date.now()
+        return this.snapshotText(tab, () => `Your current tab is ${tab.id}.`)
+      }
+      if (action === 'close') {
+        this.tabs.delete(tab.id)
+        if (this.chosen.get(root) === tab.id) this.chosen.delete(root)
+        await tab.close()
+        return text(`Closed ${tab.id}.`)
+      }
+      return refuse(`Unknown action "${action}": use list, select or close.`)
+    }
+
+    const tab = this.tabFor(root, input)
+    if (!(tab instanceof HeadlessTab)) return tab
+    if (tab.dialog && !DIALOG_SAFE.has(tool)) return refuse(dialogNote(tab))
+    if (tool !== 'browser_console' && tool !== 'browser_network') tab.agentAt = Date.now()
+
+    switch (tool) {
+      case 'browser_navigate': {
+        const action = str(input.action) as 'back' | 'forward' | 'reload' | undefined
+        const url = str(input.url)
+        if (!action && !url) return refuse('Pass url, or action back, forward or reload.')
+        if (action && !['back', 'forward', 'reload'].includes(action)) return refuse(`Unknown action "${action}": use back, forward or reload.`)
+        const wait = parseWaitInput(input.waitFor as Record<string, unknown> | undefined, null)
+        if (wait && 'error' in wait) return refuse(wait.error)
+        if (url) {
+          tab.addSecret(str(input.redact))
+          tab.noteSecretsIn(url)
+          const error = await tab.load(url)
+          if (error) return refuse(`${tab.redact(url)} did not load: ${error}`)
+        } else {
+          const error = await tab.go(action!)
+          if (error) return refuse(error.startsWith('There is no page') ? error : `${action} did not load: ${error}`)
+        }
+        return this.snapshotText(tab, () => `Now at ${tab.url}.`, wait)
+      }
+      case 'browser_snapshot': {
+        const wait = parseWaitInput(input.waitFor as Record<string, unknown> | undefined, (ref) => tab.nodeForRef(ref))
+        if (wait && 'error' in wait) return refuse(wait.error)
+        return this.snapshotText(tab, () => `Tab ${tab.id}.`, wait)
+      }
+      case 'browser_wait_for': {
+        const time = Number(input.time)
+        if (Number.isFinite(time) && time > 0 && input.text === undefined && input.textGone === undefined && !input.selector && !input.ref && input.networkIdle !== true) {
+          const ms = Math.min(time * 1000, WAIT_MAX_MS)
+          await new Promise((r) => setTimeout(r, ms))
+          return this.snapshotText(tab, () => `Waited ${seconds(ms)}.`)
+        }
+        const wait = parseWaitInput(input, (ref) => tab.nodeForRef(ref))
+        if (!wait) return refuse('Say what to wait for: text, textGone, selector or ref (with state visible or hidden), networkIdle, or time in seconds.')
+        if ('error' in wait) return refuse(wait.error)
+        return this.snapshotText(tab, () => `Tab ${tab.id}.`, wait)
+      }
+      case 'browser_click': {
+        const node = await this.nodeFor(tab, str(input.ref))
+        if (typeof node !== 'number') return node
+        const label = tab.snapshot?.labels.get(str(input.ref)!)
+        const element = this.elementName(tab, input, 'the element')
+        // Following a link is navigating, which is free; any other click outside the project is acting.
+        if (label?.role !== 'link') {
+          const no = this.permitted(tab, `click ${element}`, node)
+          if (no) return no
+        }
+        await tab.click(node, { double: input.doubleClick === true, button: (str(input.button) as 'left' | 'right' | 'middle' | undefined) ?? 'left' })
+        return this.snapshotText(tab, () => `Clicked ${element}.`)
+      }
+      case 'browser_type': {
+        const node = await this.nodeFor(tab, str(input.ref))
+        if (typeof node !== 'number') return node
+        if (await tab.isPassword(node)) return refuse('That is a password field. Never type a password: ask the person to sign in, then take a new snapshot.')
+        const element = this.elementName(tab, input, 'the field')
+        if (input.submit === true) {
+          const no = this.permitted(tab, 'submit the form', node)
+          if (no) return no
+        }
+        await tab.type(node, typeof input.text === 'string' ? input.text : '', { clear: input.clear !== false, submit: input.submit === true })
+        return this.snapshotText(tab, () => `Typed into ${element}${input.submit === true ? ' and pressed Enter' : ''}.`)
+      }
+      case 'browser_press_key': {
+        const press = parseKey(typeof input.key === 'string' ? input.key : '')
+        if ('error' in press) return refuse(press.error)
+        let node: number | undefined
+        if (str(input.ref)) {
+          const found = await this.nodeFor(tab, str(input.ref))
+          if (typeof found !== 'number') return found
+          if (press.text && (await tab.isPassword(found))) return refuse('That is a password field. Never type a password: ask the person to sign in.')
+          node = found
+        }
+        const keyName = str(input.key)!
+        // Enter and Space submit forms and press buttons: outside the project, that is acting.
+        if (press.key === 'Enter' || press.key === ' ') {
+          const no = this.permitted(tab, `press ${keyName}`, node)
+          if (no) return no
+        }
+        const where = node !== undefined ? ` in ${this.elementName(tab, input, 'the element')}` : ''
+        await tab.pressKey(press, node)
+        return this.snapshotText(tab, () => `Pressed ${keyName}${where}.`)
+      }
+      case 'browser_select_option': {
+        const node = await this.nodeFor(tab, str(input.ref))
+        if (typeof node !== 'number') return node
+        const values = Array.isArray(input.values) ? input.values.filter((v): v is string => typeof v === 'string') : typeof input.value === 'string' ? [input.value] : []
+        if (!values.length) return refuse('values is required: the options to choose, by value or by label.')
+        const element = this.elementName(tab, input, 'the list')
+        const chosen = await tab.selectOptions(node, values)
+        if ('error' in chosen) return refuse(`Could not choose in ${element}: ${chosen.error}.`)
+        return this.snapshotText(tab, () => `Chose ${chosen.values.map((v) => JSON.stringify(v)).join(', ')} in ${element}.`)
+      }
+      case 'browser_hover': {
+        const node = await this.nodeFor(tab, str(input.ref))
+        if (typeof node !== 'number') return node
+        const element = this.elementName(tab, input, 'the element')
+        await tab.hover(node)
+        return this.snapshotText(tab, () => `Hovering over ${element}.`)
+      }
+      case 'browser_evaluate': {
+        const expression = str(input.expression) ?? str(input.function)
+        if (!expression) return refuse('expression is required: a JavaScript expression, or a function such as () => document.title.')
+        // With a ref, the script runs in the frame that element is in (an App embedded in Studio), given the element.
+        let node: number | undefined
+        if (str(input.ref)) {
+          const found = await this.nodeFor(tab, str(input.ref))
+          if (typeof found !== 'number') return found
+          node = found
+        }
+        const no = this.permitted(tab, 'run script in the page', node)
+        if (no) return no
+        const result = await tab.evaluateInPage(expression, EVALUATE_CAP, 10_000, node)
+        if (tab.dialog) return refuse(`The script opened a dialog. ${dialogNote(tab)}`)
+        if ('error' in result) return refuse(`The script failed: ${result.error}`)
+        const clipped = result.length > result.json.length ? ` (the first ${EVALUATE_CAP.toLocaleString('en')} of ${result.length.toLocaleString('en')} characters)` : ''
+        return text(`Result, ${result.type}${clipped}:\n${tab.redact(result.json)}`)
+      }
+      case 'browser_set_viewport': {
+        const device = str(input.device)
+        const width = Number(input.width)
+        const height = Number(input.height)
+        if (Number.isFinite(width) && width > 0) {
+          if (width < 240 || width > 3840) return refuse('width must be between 240 and 3840.')
+          if (Number.isFinite(height) && height > 0 && (height < 240 || height > 4320)) return refuse('height must be between 240 and 4320.')
+          await tab.setSize({ width: Math.round(width), height: Number.isFinite(height) && height > 0 ? Math.round(height) : null })
+        } else if (device && device in DEVICE_SIZES) {
+          await tab.setSize({ device: device as DeviceWidth })
+        } else return refuse('Pass device (desktop, tablet or phone), or width (and height) in pixels.')
+        await tab.settle(2_000, 200)
+        const now = await tab.viewportWidth()
+        const label = tab.viewport ? `${tab.viewport.width}${tab.viewport.height ? `×${tab.viewport.height}` : ''} px` : DEVICE_SIZES[tab.device].label
+        return text(`Tab ${tab.id} lays out at ${label}; the page is ${now} px wide now.`)
+      }
+      case 'browser_handle_dialog': {
+        const dialog = tab.dialog
+        if (!dialog) return refuse('No dialog is open in this tab.')
+        const accept = input.accept !== false
+        if (accept && dialog.type !== 'alert') {
+          const no = this.permitted(tab, `answer OK to "${dialog.message.slice(0, 120)}"`)
+          if (no) return no
+        }
+        try {
+          await tab.answerDialog(accept, typeof input.promptText === 'string' ? input.promptText : undefined)
+        } catch (err) {
+          return refuse(`The dialog could not be answered (${(err as Error).message}).`)
+        }
+        return this.snapshotText(tab, () => `${accept ? 'Accepted' : 'Dismissed'} the ${dialog.type} ${JSON.stringify(dialog.message.slice(0, 200))}.`)
+      }
+      case 'browser_screenshot': {
+        if (!tab.dialog) await tab.settle(3_000, 300)
+        const data = await tab.screenshot({ fullPage: input.fullPage === true })
+        return { content: [{ type: 'text', text: `Tab ${tab.id}: ${tab.url}` }, { type: 'image', data, mimeType: 'image/png' }], meta: { tabId: tab.id, url: tab.url, root: tab.root } }
+      }
+      case 'browser_console': {
+        const entries = tab.console.filter((c) => input.onlyErrors !== true || c.level === 'error' || c.level === 'warning').slice(-200)
+        return text(entries.length ? entries.map((c) => `[${c.level}] ${c.text}${c.source ? ` (${c.source})` : ''}`).join('\n') : 'No console messages.')
+      }
+      case 'browser_network': {
+        return text(networkText([...tab.network.values()], input.onlyFailed === true))
+      }
+      default:
+        return refuse(`Unknown browser tool "${tool}".`)
+    }
+  }
+}
+
+let processSession: HeadlessBrowserSession | null = null
+
+/** The MCP server process's one headless browser session. */
+export function headlessSession(): HeadlessBrowserSession {
+  processSession ??= new HeadlessBrowserSession()
+  return processSession
+}