@opentiny/next-sdk 0.4.0 → 0.4.1

package/page-tools/handlers/executeJavascript.ts

@@ -0,0 +1,22 @@
+import type { ActionContext } from '../context'
+
+export async function handleExecuteJavascript(args: any, ctx: ActionContext) {
+  if (!args.script) return ctx.errContent('脚本执行异常: 缺少javascript代码')
+  // eslint-disable-next-line no-new-func
+  // 方式1：将脚本包裹在 async IIFE 中执行，允许 return 语句
+  let result = await new Function(`return (async () => { ${args.script} })()`)()
+  // 方式2：若 result 为 undefined（脚本没有 return），降级尝试以表达式方式求值
+  // 场景：Agent 写了 "Array.from(...).map(...)" 但没有 return 关键字
+  if (result === undefined) {
+    try {
+      // eslint-disable-next-line no-new-func
+      result = await new Function(`return (async () => (${args.script}))()`)()
+    } catch {
+      // 表达式求值也失败（如含 await/let/const 等语句），保持 undefined
+    }
+  }
+  await ctx.pageController.hideMask()
+  return {
+    content: [{ type: 'text' as const, text: `脚本执行结果: ${JSON.stringify(result)}` }]
+  }
+}

package/page-tools/handlers/fill.ts

@@ -0,0 +1,60 @@
+import { inputTextElement } from '@page-agent/page-controller'
+import { dispatchComposedEvents, isInShadowDom, waitForDomSettled } from '../utils/dom'
+import type { ActionContext } from '../context'
+
+export async function handleFill(args: any, ctx: ActionContext) {
+  const mode = args.responseMode ?? 'diff'
+  if (args.index === undefined || typeof args.text !== 'string') return ctx.errContent('填写结果: 缺少元素索引或文本内容')
+  const el = ctx.getRefMap().get(args.index)
+  if (!el) return ctx.refreshOnStaleRef('填写', args.index)
+
+  let targetEl = el
+  if (!(targetEl instanceof HTMLInputElement) && !(targetEl instanceof HTMLTextAreaElement)) {
+    // querySelector 不穿透 shadow boundary，对 shadow host 补查其 shadowRoot
+    const innerInput = el.querySelector('input, textarea')
+      ?? (el.shadowRoot?.querySelector('input, textarea') as HTMLElement | null)
+    if (innerInput) {
+      targetEl = innerInput as HTMLElement
+    }
+  }
+
+  if (targetEl instanceof HTMLInputElement && targetEl.readOnly) {
+    targetEl.value = args.text
+    // composed:true 确保事件穿透 shadow boundary，事件委托框架可捕获
+    dispatchComposedEvents(targetEl, 'input', 'change', 'blur')
+  } else {
+    // 优先尝试标准方式（支持 contenteditable、普通 input/textarea）
+    let fillSuccess = false
+    try {
+      await inputTextElement(targetEl, args.text)
+      fillSuccess = true
+    } catch (fillErr) {
+      // inputTextElement 失败时（如 Angular/React 组件包装的密码框），
+      // 降级使用原生 input value 描述符触发框架变更检测
+      const nativeInputValueSetter = Object.getOwnPropertyDescriptor(
+        (targetEl instanceof HTMLTextAreaElement ? HTMLTextAreaElement : HTMLInputElement).prototype,
+        'value'
+      )?.set
+      if (nativeInputValueSetter && (targetEl instanceof HTMLInputElement || targetEl instanceof HTMLTextAreaElement)) {
+        nativeInputValueSetter.call(targetEl, args.text)
+        // 发送完整事件序列，触发 Angular ngModel / React 合成事件的变更检测
+        targetEl.dispatchEvent(new Event('focus', { bubbles: true, composed: true }))
+        targetEl.dispatchEvent(new Event('input', { bubbles: true, composed: true }))
+        targetEl.dispatchEvent(new Event('change', { bubbles: true, composed: true }))
+        targetEl.dispatchEvent(new Event('blur', { bubbles: true, composed: true }))
+        fillSuccess = true
+      }
+    }
+    if (!fillSuccess) {
+      return ctx.errContent(`填写结果: 无法填写元素 ${args.index}，元素不是 input、textarea 或 contenteditable`)
+    }
+    // shadow DOM 内元素：inputTextElement 派发的合成事件 composed:false，
+    // 补发 composed:true 事件确保事件委托框架（如 React）能收到
+    if (isInShadowDom(targetEl)) {
+      dispatchComposedEvents(targetEl, 'input', 'change')
+    }
+  }
+  // 等待框架异步插入的校验错误/条件渲染内容稳定后再采集状态
+  await waitForDomSettled()
+  return ctx.buildBrowserStateResponse(mode)
+}

package/page-tools/handlers/scroll.ts

@@ -0,0 +1,66 @@
+import { getScrollInfo, waitForScrollEnd } from '../utils/scroll'
+import type { ActionContext } from '../context'
+
+export async function handleScroll(args: any, ctx: ActionContext) {
+  const mode = args.responseMode ?? 'diff'
+  if (args.down === undefined && args.right === undefined) return ctx.errContent('滚动结果: 缺少滚动方向参数')
+
+  // 确定滚动目标（有 index 时滚动该元素容器，否则滚动整个文档）
+  let scrollTarget: Element | Window = window
+  if (args.index !== undefined) {
+    const el = ctx.getRefMap().get(args.index)
+    if (!el) return ctx.refreshOnStaleRef('滚动', args.index)
+    scrollTarget = el
+  }
+
+  // 滚动前快照
+  const before = getScrollInfo(scrollTarget)
+
+  if (args.right !== undefined) {
+    const pixels = args.pixels ?? 300
+    scrollTarget.scrollBy({ left: args.right ? pixels : -pixels, behavior: 'smooth' })
+  } else {
+    const pixels = args.pixels ?? Math.round((args.numPages ?? 1) * window.innerHeight)
+    scrollTarget.scrollBy({ top: args.down ? pixels : -pixels, behavior: 'smooth' })
+  }
+
+  // 等待滚动动画完成后再采集状态（scrollend + 超时兜底）
+  await waitForScrollEnd(scrollTarget)
+
+  // 滚动后快照，计算实际位移
+  const after = getScrollInfo(scrollTarget)
+  const deltaY = Math.round(after.scrollY - before.scrollY)
+  const deltaX = Math.round(after.scrollX - before.scrollX)
+  const isHorizontal = args.right !== undefined
+
+  let scrollMsg: string
+  if (Math.abs(deltaY) < 1 && Math.abs(deltaX) < 1) {
+    // 实际未发生位移，说明已到达边界
+    if (isHorizontal) {
+      scrollMsg = args.right ? '⚠️ 已到达右边界，无法继续向右滚动' : '⚠️ 已到达左边界，无法继续向左滚动'
+    } else {
+      scrollMsg = args.down ? '⚠️ 已到达底部，无法继续向下滚动' : '⚠️ 已到达顶部，无法继续向上滚动'
+    }
+  } else {
+    const axis = isHorizontal ? `水平滚动 ${deltaX}px` : `垂直滚动 ${deltaY}px`
+    let boundary = ''
+    if (isHorizontal) {
+      if (args.right && after.atRight) boundary = '，已到达右边界'
+      else if (!args.right && after.atLeft) boundary = '，已到达左边界'
+    } else {
+      if (args.down && after.atBottom) boundary = '，已到达底部'
+      else if (!args.down && after.atTop) boundary = '，已到达顶部'
+    }
+    scrollMsg = `✅ ${axis}${boundary}`
+  }
+
+  // 附加位置信息（参考 page-agent getPageInfo 格式）
+  const posInfo = args.right !== undefined
+    ? `当前水平滚动位置: scrollX=${Math.round(after.scrollX)}px，左侧 ${after.pagesAbove.toFixed(1)} 屏，右侧 ${after.pagesBelow.toFixed(1)} 屏`
+    : `当前滚动位置: scrollY=${Math.round(after.scrollY)}px，上方 ${after.pagesAbove.toFixed(1)} 屏，下方 ${after.pagesBelow.toFixed(1)} 屏`
+  const scrollResult = `[滚动结果] ${scrollMsg}\n${posInfo}`
+
+  const stateResult = await ctx.buildBrowserStateResponse(mode)
+  stateResult.content[0].text = `${scrollResult}\n\n${stateResult.content[0].text}`
+  return stateResult
+}

package/page-tools/handlers/searchTree.ts

@@ -0,0 +1,30 @@
+import { searchA11yTree } from '../a11y-tree'
+import { detectPageDialog, detectValidationErrors } from '../utils/dom'
+import { DEFAULT_ERROR_SELECTORS } from '../constants'
+import type { ActionContext } from '../context'
+
+export async function handleSearchTree(args: any, ctx: ActionContext) {
+  if (!args.query) return ctx.errContent('搜索失败: 缺少 query 参数')
+  const blacklist = (window.__webmcpcli_interactiveBlacklist ?? []) as Element[]
+  const whitelist = (window.__webmcpcli_interactiveWhitelist ?? []) as Element[]
+  const exposedAttributes = (window.__webmcpcli_exposedAttributes ?? []) as string[]
+  const result = searchA11yTree(args.query, document.body, blacklist, whitelist, {
+    contextLines: args.contextLines,
+    maxMatches: args.maxMatches,
+    exposedAttributes,
+    errorSelectors: (window.__webmcpcli_errorSelectors ?? DEFAULT_ERROR_SELECTORS).join(', '),
+  })
+  
+  ctx.setRefMap(result.refMap)
+  ctx.stateCache.update(window.location.href, result.yaml)
+  
+  await ctx.pageController.hideMask()
+  // 检测页面弹窗/遮罩层，帮助 AI 发现可能遮挡目标的确认弹窗
+  const dialogAlert = detectPageDialog()
+  // 检测表单校验错误，提醒 AI 优先修复
+  const validationErrors = detectValidationErrors()
+  const alerts = `${dialogAlert}${validationErrors}`
+  return {
+    content: [{ type: 'text' as const, text: alerts ? `${alerts}\n${result.text}` : result.text }]
+  }
+}

package/page-tools/handlers/select.ts

@@ -0,0 +1,34 @@
+import { selectOptionElement } from '@page-agent/page-controller'
+import { dispatchComposedEvents, isInShadowDom, waitForDomSettled } from '../utils/dom'
+import type { ActionContext } from '../context'
+
+export async function handleSelect(args: any, ctx: ActionContext) {
+  const mode = args.responseMode ?? 'diff'
+  if (args.index === undefined || typeof args.text !== 'string') return ctx.errContent('选择结果: 缺少元素索引或文本内容')
+  let el = ctx.getRefMap().get(args.index) as HTMLSelectElement | HTMLElement | undefined
+  if (!el) return ctx.refreshOnStaleRef('选择', args.index)
+  // 若 ref 指向 shadow host，解析其内部真正的 <select>（与 fill 一致）
+  let targetSelect: HTMLSelectElement | null = null
+  if (el instanceof HTMLSelectElement) {
+    targetSelect = el
+  } else {
+    const innerSelect = el.querySelector('select')
+      ?? (el.shadowRoot?.querySelector('select') as HTMLSelectElement | null)
+    if (innerSelect) {
+      targetSelect = innerSelect
+    }
+  }
+
+  if (!targetSelect) {
+    return ctx.errContent(`选择结果: 无法选择元素 ${args.index}，未找到对应的 <select> 元素`)
+  }
+
+  await selectOptionElement(targetSelect, args.text)
+  // shadow DOM 内元素：补发 composed:true 的 change 事件
+  if (isInShadowDom(targetSelect)) {
+    dispatchComposedEvents(targetSelect, 'change')
+  }
+  // 等待框架异步插入的校验错误/条件渲染内容稳定后再采集状态
+  await waitForDomSettled()
+  return ctx.buildBrowserStateResponse(mode)
+}

package/page-tools/page-agent-highlight/index.ts

@@ -0,0 +1,198 @@
+import { type RefMap } from '../a11y-tree'
+
+export const HIGHLIGHT_CONTAINER_ID = 'webmcpcli-highlight-container'
+export const HIGHLIGHT_CONTAINER_STYLE_ID = 'webmcpcli-highlight-container-style'
+const colors = [
+  '#FF0000',
+  '#00FF00',
+  '#0000FF',
+  '#FFA500',
+  '#800080',
+  '#008080',
+  '#FF69B4',
+  '#4B0082',
+  '#FF4500',
+  '#2E8B57',
+  '#DC143C',
+  '#4682B4'
+]
+// border + label opacity
+const opacity = Math.floor(0.3 * 255)
+  .toString(16)
+  .padStart(2, '0')
+
+const injectStyles = `
+      #${HIGHLIGHT_CONTAINER_ID} {
+        position: fixed;
+        pointer-events: none;
+        top: 0;
+        left: 0;
+        width: 100%;
+        height: 100%;
+        z-index: 2147483640;
+        background-color: transparent;
+      }
+
+      #${HIGHLIGHT_CONTAINER_ID} div {
+        position: fixed;
+        pointer-events: none;
+        background-color: transparent;
+        text-align: right;
+        box-sizing: border-box;
+        border-radius: 4px;
+      }
+
+      #${HIGHLIGHT_CONTAINER_ID} div span {
+        display: inline-block;
+        color: #fff;
+        padding: 0 2px;
+        border-radius: 4px;
+      }
+  `
+
+function isElementViewport(element: HTMLElement) {
+  const rect = element.getBoundingClientRect()
+  return (
+    rect.top >= 0 &&
+    rect.left >= 0 &&
+    rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
+    rect.right <= (window.innerWidth || document.documentElement.clientWidth)
+  )
+}
+
+const doHighlight = (refMap: RefMap, parentIframe: HTMLIFrameElement | null = null) => {
+  // 1、保证注入脚本
+  if (!document.head.querySelector(`#${HIGHLIGHT_CONTAINER_STYLE_ID}`)) {
+    const style = document.head.appendChild(document.createElement('style'))
+    style.id = HIGHLIGHT_CONTAINER_STYLE_ID
+    style.textContent = injectStyles
+    document.head.appendChild(style)
+  }
+
+  // 2、保证contrainer( container 在 a11y 时已加入黑名单)
+  let container = document.getElementById(HIGHLIGHT_CONTAINER_ID)
+  if (!container) {
+    container = document.createElement('div')
+    container.id = HIGHLIGHT_CONTAINER_ID
+
+    document.body.appendChild(container)
+  } else {
+    container.innerHTML = ''
+  }
+
+  let iframeOffset = { x: 0, y: 0 }
+  if (parentIframe) {
+    const iframeRect = parentIframe.getBoundingClientRect() // Keep getBoundingClientRect for iframe offset
+    iframeOffset.x = iframeRect.left
+    iframeOffset.y = iframeRect.top
+  }
+  const fragment = document.createDocumentFragment()
+  const overlays = []
+
+  // 3、 遍历 refMap
+  for (const [index, el] of refMap.entries()) {
+    if (!isElementViewport(el)) continue
+
+    const color = colors[index % colors.length]
+    const textColor = color + opacity
+
+    const rect = el.getBoundingClientRect()
+    if (rect.width === 0 || rect.height === 0) continue
+
+    const overlay = document.createElement('div')
+    overlay.style.border = `2px solid ${textColor}`
+
+    const top = rect.top + iframeOffset.y
+    const left = rect.left + iframeOffset.x
+
+    overlay.style.top = `${top}px`
+    overlay.style.left = `${left}px`
+    overlay.style.width = `${rect.width}px`
+    overlay.style.height = `${rect.height}px`
+
+    // 文字
+    const text = document.createElement('span')
+    text.style.backgroundColor = textColor
+    text.style.height = `${rect.height - 4}px`
+    text.style.lineHeight = `${rect.height - 4}px`
+    text.textContent = index.toString()
+    overlay.appendChild(text)
+
+    fragment.appendChild(overlay)
+    overlays.push({ element: overlay, initialRect: rect })
+  }
+
+  container.appendChild(fragment)
+}
+
+export const highlight = (refMap: RefMap, parentIframe: HTMLIFrameElement | null = null) => {
+  doHighlight(refMap, parentIframe)
+
+  // 4、监听滚动和resize事件，更新位置
+  const updatePositions = () => doHighlight(refMap, parentIframe)
+  const throttleFunction = (func: (...args: any[]) => void, delay: number) => {
+    let lastCall = 0
+    return (...args: any[]) => {
+      const now = performance.now()
+      if (now - lastCall < delay) return
+      lastCall = now
+      return func(...args)
+    }
+  }
+
+  const throttledUpdatePositions = throttleFunction(updatePositions, 16)
+  window.addEventListener('scroll', throttledUpdatePositions, true)
+  window.addEventListener('resize', throttledUpdatePositions)
+
+  const cleanupFn = () => {
+    window.removeEventListener('scroll', throttledUpdatePositions, true)
+    window.removeEventListener('resize', throttledUpdatePositions)
+  }
+  const win = window as any
+  ;(win._highlightCleanupFunctions = win._highlightCleanupFunctions || []).push(cleanupFn)
+}
+
+/** 移除高亮， 可反复调用 */
+export const unhighlight = () => {
+  document.getElementById(HIGHLIGHT_CONTAINER_ID)?.remove()
+
+  const cleanupFunctions = (window as any)._highlightCleanupFunctions || []
+  for (const cleanup of cleanupFunctions) {
+    if (typeof cleanup === 'function') {
+      cleanup()
+    }
+  }
+
+  ;(window as any)._highlightCleanupFunctions = []
+}
+
+/** 全局监听变化，随时准备移除高亮, 可反复调用 */
+export const globalRemoveListener = () => {
+  if ((window as any).__registerGlobalRemoveListener) return
+  ;(window as any).__registerGlobalRemoveListener = true
+  window.addEventListener('popstate', () => {
+    unhighlight()
+  })
+  window.addEventListener('hashchange', () => {
+    unhighlight()
+  })
+  window.addEventListener('beforeunload', () => {
+    unhighlight()
+  })
+
+  const navigation = (window as any).navigation
+  if (navigation && typeof navigation.addEventListener === 'function') {
+    navigation.addEventListener('navigate', () => {
+      unhighlight()
+    })
+  } else {
+    // 定时器
+    let currentUrl = window.location.href
+    setInterval(() => {
+      if (window.location.href !== currentUrl) {
+        currentUrl = window.location.href
+        unhighlight()
+      }
+    }, 500)
+  }
+}

package/page-tools/page-agent-mask/SimulatorMask.module.css

@@ -0,0 +1,14 @@
+.webmcp-page-agent-wrapper {
+	position: fixed;
+	inset: 0;
+	z-index: 2147483641;
+	/* 确保在所有元素之上，除了 panel */
+	cursor: wait;
+	overflow: hidden;
+
+	display: none;
+}
+
+.webmcp-page-agent-wrapper.visible {
+	display: block;
+}