@frontera-sdk/cli 1.52.38 → 1.52.40

package/src/mcp/browser/frames.ts

@@ -0,0 +1,336 @@
+import type { AxNode, FrameContent } from './ax-snapshot'
+import { selectorVisible, textPresent } from './page-scripts'
+
+/**
+ * The frames of one page, for the coding agent's browser: an App embedded in
+ * Studio is an <iframe> on another origin (`*.apps.localhost`), and Chrome
+ * renders a frame from another site in a process of its own, reached through
+ * a DevTools session of its own. A snapshot reads every frame's accessibility
+ * tree into the page's, and a ref names an element in any of them, so the
+ * agent clicks, types and waits inside an embedded App as it does on the page.
+ *
+ * Shared by both browsers the coding agent drives: Forge's (Electron, whose
+ * debugger attaches to out-of-process frames itself) and the headless one
+ * `frontera mcp` runs without Forge (Playwright, a session per such frame).
+ * Each says how it reaches a frame's session (`FrameSessions`); the rest is here.
+ *
+ * Refs stay what they were on the page itself: an element of the main
+ * document is named by its DOM node (`backendNodeId`). An element in a frame
+ * is named by a handle (a negative number) that maps to its frame and node,
+ * since node ids are only unique within one renderer process.
+ */
+
+export type Cdp = <T = Record<string, unknown>>(method: string, params?: Record<string, unknown>) => Promise<T>
+
+export interface FrameSessions {
+  /** The page's own session. */
+  readonly root: Cdp
+  /** The session of a frame Chrome renders in a process of its own (its target id is its frame id), or null when it renders in its parent's. */
+  outOfProcess(frameId: string): Promise<Cdp | null>
+}
+
+export interface PageFrame {
+  /** The DevTools frame id; null for the page's main frame until it is read. */
+  id: string | null
+  /** The session the frame's DOM is reached through. */
+  cdp: Cdp
+  /** The frame is its session's top (the main frame, or one in a process of its own): its coordinates start at its own corner. */
+  sessionTop: boolean
+  parent: PageFrame | null
+  /** The <iframe> element in the parent's document. */
+  owner: number | null
+  /** The frame's address, as its document says (`location.href`). */
+  href: string
+  world: number | null
+  depth: number
+}
+
+/** How deep frames are read, and how many at most: a page of ads must not flood the snapshot. */
+export const MAX_FRAME_DEPTH = 4
+export const MAX_FRAMES = 24
+
+const GROUP = 'frontera-frames'
+
+/** An address without its query or fragment, which can carry a sign-in token: what a snapshot shows of a frame. */
+export function shownAddress(href: string): string {
+  try {
+    const u = new URL(href)
+    if (u.protocol === 'about:' || u.protocol === 'data:' || u.protocol === 'blob:') return u.protocol === 'about:' ? href.split('?')[0]! : `${u.protocol}…`
+    return `${u.origin}${u.pathname}`
+  } catch {
+    return href.split(/[?#]/)[0] ?? href
+  }
+}
+
+export class PageFrames {
+  readonly main: PageFrame
+  private byId = new Map<string, PageFrame>()
+  private handles = new Map<number, { frame: PageFrame; node: number }>()
+  private handleKeys = new Map<string, number>()
+  private nextHandle = 0
+
+  constructor(
+    private readonly sessions: FrameSessions,
+    private readonly worldName: string,
+  ) {
+    this.main = { id: null, cdp: sessions.root, sessionTop: true, parent: null, owner: null, href: '', world: null, depth: 0 }
+  }
+
+  /** The page went to another document: its frames, their nodes and worlds are gone. */
+  reset(): void {
+    this.byId.clear()
+    this.handles.clear()
+    this.handleKeys.clear()
+    this.main.world = null
+    this.main.id = null
+  }
+
+  /** A frame went to another document: its world is gone (its nodes go with it, and their refs say so when used). */
+  frameNavigated(frameId: string): void {
+    const f = this.byId.get(frameId)
+    if (f) f.world = null
+  }
+
+  /** The number that names an element: its DOM node on the main document, a handle in a frame. */
+  handleFor(frame: PageFrame, node: number): number {
+    if (frame === this.main) return node
+    const key = `${frame.id}:${node}`
+    const known = this.handleKeys.get(key)
+    if (known !== undefined) return known
+    const handle = --this.nextHandle
+    this.handleKeys.set(key, handle)
+    this.handles.set(handle, { frame, node })
+    return handle
+  }
+
+  /** The frame and DOM node a handle names. */
+  locate(handle: number): { frame: PageFrame; node: number } {
+    if (handle >= 0) return { frame: this.main, node: handle }
+    const at = this.handles.get(handle)
+    if (!at) throw new Error('the element is gone with its frame: take a new browser_snapshot')
+    return at
+  }
+
+  /** Whether the element is in a frame rather than on the page itself. */
+  inFrame(handle: number): boolean {
+    return handle < 0
+  }
+
+  /** The address of the frame the element is in. */
+  addressOf(handle: number): string | null {
+    if (handle >= 0) return null
+    return this.handles.get(handle)?.frame.href ?? null
+  }
+
+  /** The frame an <iframe> element shows, its session found; null when it shows none. */
+  async childFrame(parent: PageFrame, owner: number): Promise<PageFrame | null> {
+    const { node } = await parent.cdp<{ node: { frameId?: string; contentDocument?: unknown } }>('DOM.describeNode', { backendNodeId: owner })
+    const frameId = node.frameId
+    if (!frameId) return null
+    const own = await this.sessions.outOfProcess(frameId)
+    const cdp = own ?? parent.cdp
+    const known = this.byId.get(frameId)
+    if (known && known.cdp === cdp && known.parent === parent) {
+      known.owner = owner
+      return known
+    }
+    const frame: PageFrame = { id: frameId, cdp, sessionTop: !!own, parent, owner, href: '', world: null, depth: parent.depth + 1 }
+    this.byId.set(frameId, frame)
+    return frame
+  }
+
+  private async frameIdOf(frame: PageFrame): Promise<string> {
+    if (frame.id) return frame.id
+    const tree = await frame.cdp<{ frameTree: { frame: { id: string } } }>('Page.getFrameTree')
+    frame.id = tree.frameTree.frame.id
+    return frame.id
+  }
+
+  /** The frame's isolated world, where the browser's own scripts run unseen by the page's. */
+  async world(frame: PageFrame): Promise<number> {
+    if (frame.world !== null) return frame.world
+    const frameId = await this.frameIdOf(frame)
+    const world = await frame.cdp<{ executionContextId: number }>('Page.createIsolatedWorld', { frameId, worldName: this.worldName, grantUniveralAccess: false })
+    frame.world = world.executionContextId
+    return frame.world
+  }
+
+  /** The browser's own expression in a frame's isolated world; null when the frame cannot answer. */
+  async evaluate<T>(frame: PageFrame, expression: string): Promise<T | null> {
+    for (let attempt = 0; attempt < 2; attempt++) {
+      try {
+        const res = await frame.cdp<{ result: { value?: T } }>('Runtime.evaluate', { expression, contextId: await this.world(frame), returnByValue: true, awaitPromise: true })
+        return res.result.value ?? null
+      } catch {
+        // A frame that navigated took its world with it: make another, once.
+        frame.world = null
+      }
+    }
+    return null
+  }
+
+  /** Call the browser's function on an element, in its frame's isolated world (or the page's own world). */
+  async callOn<T>(handle: number, fn: string, args: unknown[] = [], opts: { mainWorld?: boolean } = {}): Promise<T | null> {
+    const { frame, node } = this.locate(handle)
+    const resolve = async () => frame.cdp<{ object: { objectId?: string } }>('DOM.resolveNode', opts.mainWorld ? { backendNodeId: node } : { backendNodeId: node, executionContextId: await this.world(frame) })
+    let resolved: { object: { objectId?: string } }
+    try {
+      resolved = await resolve()
+    } catch (err) {
+      if (opts.mainWorld) throw err
+      frame.world = null
+      resolved = await resolve()
+    }
+    if (!resolved.object.objectId) return null
+    const res = await frame.cdp<{ result: { value?: T } }>('Runtime.callFunctionOn', { objectId: resolved.object.objectId, functionDeclaration: fn, arguments: args.map((value) => ({ value })), returnByValue: true })
+    return res.result.value ?? null
+  }
+
+  /** Where a frame's own coordinates start, on the page. */
+  private async offset(frame: PageFrame): Promise<{ x: number; y: number }> {
+    if (!frame.parent) return { x: 0, y: 0 }
+    const above = await this.offset(frame.parent)
+    // A frame in its parent's process reports page coordinates already.
+    if (!frame.sessionTop) return above
+    const { model } = await frame.parent.cdp<{ model: { content: number[] } }>('DOM.getBoxModel', { backendNodeId: frame.owner })
+    return { x: above.x + model.content[0]!, y: above.y + model.content[1]! }
+  }
+
+  /** The middle of an element on the page, scrolled into view (and its frames with it). */
+  async center(handle: number): Promise<{ x: number; y: number }> {
+    const { frame, node } = this.locate(handle)
+    const chain: PageFrame[] = []
+    for (let f: PageFrame | null = frame; f?.parent; f = f.parent) chain.unshift(f)
+    for (const f of chain) await f.parent!.cdp('DOM.scrollIntoViewIfNeeded', { backendNodeId: f.owner }).catch(() => {})
+    await frame.cdp('DOM.scrollIntoViewIfNeeded', { backendNodeId: node }).catch(() => {})
+    const { quads } = await frame.cdp<{ quads: number[][] }>('DOM.getContentQuads', { backendNodeId: node })
+    const q = quads[0]
+    if (!q) throw new Error('the element is not on screen (hidden, or zero size)')
+    const at = await this.offset(frame)
+    return { x: at.x + (q[0]! + q[2]! + q[4]! + q[6]!) / 4, y: at.y + (q[1]! + q[3]! + q[5]! + q[7]!) / 4 }
+  }
+
+  /** Focus an element; in a frame through DevTools, which moves the page's focus into that frame (a script's focus() cannot). */
+  async focus(handle: number, script: string): Promise<void> {
+    const { frame, node } = this.locate(handle)
+    if (frame !== this.main) await frame.cdp('DOM.focus', { backendNodeId: node }).catch(() => {})
+    await this.callOn(handle, script)
+  }
+
+  private async readHref(frame: PageFrame): Promise<void> {
+    const href = await this.evaluate<string>(frame, 'location.href')
+    if (typeof href === 'string') frame.href = href
+  }
+
+  /** The page's accessibility tree with every frame's read into it, for the snapshot. */
+  async snapshotTree(): Promise<{ nodes: AxNode[]; frames: Map<string, FrameContent | { error: string }> }> {
+    const budget = { frames: MAX_FRAMES }
+    const { nodes } = await this.main.cdp<{ nodes: AxNode[] }>('Accessibility.getFullAXTree')
+    return { nodes, frames: await this.childTrees(this.main, nodes, budget) }
+  }
+
+  private async childTrees(parent: PageFrame, nodes: AxNode[], budget: { frames: number }): Promise<Map<string, FrameContent | { error: string }>> {
+    const out = new Map<string, FrameContent | { error: string }>()
+    if (parent.depth >= MAX_FRAME_DEPTH) return out
+    for (const n of nodes) {
+      if (n.ignored || n.backendDOMNodeId === undefined || String(n.role?.value ?? '') !== 'Iframe') continue
+      if (budget.frames <= 0) {
+        out.set(n.nodeId, { error: `the snapshot reads at most ${MAX_FRAMES} frames` })
+        continue
+      }
+      budget.frames--
+      try {
+        const frame = await this.childFrame(parent, n.backendDOMNodeId)
+        if (!frame) continue
+        await this.readHref(frame)
+        const tree = await frame.cdp<{ nodes: AxNode[] }>('Accessibility.getFullAXTree', frame.sessionTop ? {} : { frameId: frame.id })
+        out.set(n.nodeId, {
+          nodes: tree.nodes,
+          url: shownAddress(frame.href),
+          handleOf: (node) => this.handleFor(frame, node),
+          frames: await this.childTrees(frame, tree.nodes, budget),
+        })
+      } catch (err) {
+        out.set(n.nodeId, { error: ((err as Error).message || 'it did not answer').slice(0, 200) })
+      }
+    }
+    return out
+  }
+
+  /** The <iframe> and <frame> elements in a frame's document. */
+  private async ownersIn(frame: PageFrame): Promise<number[]> {
+    const contextId = await this.world(frame)
+    const list = await frame.cdp<{ result: { objectId?: string } }>('Runtime.evaluate', { expression: "[...document.querySelectorAll('iframe,frame')]", contextId, objectGroup: GROUP, returnByValue: false })
+    if (!list.result.objectId) return []
+    try {
+      const props = await frame.cdp<{ result: Array<{ name: string; value?: { objectId?: string } }> }>('Runtime.getProperties', { objectId: list.result.objectId, ownProperties: true })
+      const owners: number[] = []
+      for (const p of props.result) {
+        if (!/^\d+$/.test(p.name) || !p.value?.objectId) continue
+        const { node } = await frame.cdp<{ node: { backendNodeId: number } }>('DOM.describeNode', { objectId: p.value.objectId })
+        owners.push(node.backendNodeId)
+      }
+      return owners
+    } finally {
+      await frame.cdp('Runtime.releaseObjectGroup', { objectGroup: GROUP }).catch(() => {})
+    }
+  }
+
+  /** Every frame of the page now, the main one first, bounded. */
+  async all(): Promise<PageFrame[]> {
+    const out: PageFrame[] = [this.main]
+    for (let i = 0; i < out.length && out.length < MAX_FRAMES; i++) {
+      const f = out[i]!
+      if (f.depth >= MAX_FRAME_DEPTH) continue
+      const owners = await this.ownersIn(f).catch(() => [] as number[])
+      for (const owner of owners) {
+        const child = await this.childFrame(f, owner).catch(() => null)
+        if (child && !out.includes(child)) out.push(child)
+        if (out.length >= MAX_FRAMES) break
+      }
+    }
+    return out
+  }
+}
+
+/** How often a wait looks for frames again (one that opened after the wait began). */
+export const FRAMES_AGAIN_MS = 2_000
+
+/** What the agent can wait for, as both browsers read it. */
+export interface FrameWaitCondition {
+  text?: string
+  textGone?: string
+  selector?: string
+  state?: 'visible' | 'hidden'
+  ref?: number
+}
+
+/**
+ * Whether a wait's condition holds now, in any of the page's frames; a
+ * string when it can never hold (a selector that is not one). Text shows
+ * when any frame shows it and is gone when none does; a selector shows when
+ * it matches a drawn element in any frame.
+ */
+export async function conditionHolds(frames: PageFrames, list: PageFrame[], c: FrameWaitCondition, isShown: (handle: number) => Promise<boolean>): Promise<boolean | string> {
+  const anyFrame = async (expression: string) => {
+    for (const f of list) if ((await frames.evaluate<boolean>(f, expression)) === true) return true
+    return false
+  }
+  if (c.text !== undefined && !(await anyFrame(textPresent(c.text)))) return false
+  if (c.textGone !== undefined && (await anyFrame(textPresent(c.textGone)))) return false
+  const hidden = c.state === 'hidden'
+  if (c.selector !== undefined) {
+    let shown = false
+    for (const f of list) {
+      const v = await frames.evaluate<boolean | { error: string }>(f, selectorVisible(c.selector))
+      if (v && typeof v === 'object') return `"${c.selector}" is not a CSS selector: ${v.error}`
+      if (v === true) {
+        shown = true
+        break
+      }
+    }
+    if (shown === hidden) return false
+  }
+  if (c.ref !== undefined && (await isShown(c.ref)) === hidden) return false
+  return true
+}