handhold-sdk 1.0.0

package/src/agent/dom/actions.js

@@ -0,0 +1,456 @@
+/**
+ * actions.js — ported from alibaba/page-agent packages/page-controller/src/actions.ts (MIT).
+ * Audit before porting (2026-07-05): clean — dispatchEvent/innerText/execCommand only,
+ * no eval, no innerHTML, no network. This file keeps upstream logic intact because it
+ * encodes real-world compatibility work:
+ *   - full W3C pointer+mouse event order so React/Vue/antd handlers fire
+ *   - native prototype value setter so React controlled inputs update
+ *   - contenteditable Plan A (synthetic InputEvents) with verified fallback to
+ *     Plan B (execCommand) for Quill/Slate-style editors
+ * Changes vs upstream: mask CustomEvents → optional hooks (see utils.js).
+ */
+import {
+	clickPointer,
+	disablePassThrough,
+	enablePassThrough,
+	getNativeValueSetter,
+	isHTMLElement,
+	isInputElement,
+	isSelectElement,
+	isTextAreaElement,
+	movePointerToElement,
+	waitFor,
+} from './utils.js'
+
+/** Get the HTMLElement by index from a selectorMap. */
+export function getElementByIndex(selectorMap, index) {
+	const interactiveNode = selectorMap.get(index)
+	if (!interactiveNode) {
+		throw new Error(`No interactive element found at index ${index}`)
+	}
+	const element = interactiveNode.ref
+	if (!element) {
+		throw new Error(`Element at index ${index} does not have a reference`)
+	}
+	if (!isHTMLElement(element)) {
+		throw new Error(`Element at index ${index} is not an HTMLElement`)
+	}
+	return element
+}
+
+let lastClickedElement = null
+
+function blurLastClickedElement() {
+	if (lastClickedElement) {
+		lastClickedElement.dispatchEvent(new PointerEvent('pointerout', { bubbles: true }))
+		lastClickedElement.dispatchEvent(new PointerEvent('pointerleave', { bubbles: false }))
+		lastClickedElement.dispatchEvent(new MouseEvent('mouseout', { bubbles: true }))
+		lastClickedElement.dispatchEvent(new MouseEvent('mouseleave', { bubbles: false }))
+		lastClickedElement.blur()
+		lastClickedElement = null
+	}
+}
+
+/**
+ * Simulate a full click following W3C Pointer Events + UI Events spec order:
+ * pointerover/enter → mouseover/enter → pointerdown → mousedown → [focus] →
+ * pointerup → mouseup → click
+ */
+export async function clickElement(element) {
+	blurLastClickedElement()
+	lastClickedElement = element
+
+	await scrollIntoViewIfNeeded(element)
+	const frame = element.ownerDocument.defaultView?.frameElement
+	if (frame) await scrollIntoViewIfNeeded(frame)
+
+	const rect = element.getBoundingClientRect()
+	const x = rect.left + rect.width / 2
+	const y = rect.top + rect.height / 2
+
+	await movePointerToElement(element, x, y)
+	await clickPointer()
+
+	await waitFor(0.1)
+
+	// Hit-test to find the deepest element at click coordinates, matching
+	// real browser behavior where events target the innermost element.
+	const doc = element.ownerDocument
+	await enablePassThrough()
+	const hitTarget = doc.elementFromPoint ? doc.elementFromPoint(x, y) : null
+	await disablePassThrough()
+	const target = isHTMLElement(hitTarget) && element.contains(hitTarget) ? hitTarget : element
+
+	const pointerOpts = { bubbles: true, cancelable: true, clientX: x, clientY: y, pointerType: 'mouse' }
+	const mouseOpts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 }
+
+	// Hover — pointer events first, then mouse events (spec order)
+	target.dispatchEvent(new PointerEvent('pointerover', pointerOpts))
+	target.dispatchEvent(new PointerEvent('pointerenter', { ...pointerOpts, bubbles: false }))
+	target.dispatchEvent(new MouseEvent('mouseover', mouseOpts))
+	target.dispatchEvent(new MouseEvent('mouseenter', { ...mouseOpts, bubbles: false }))
+
+	// Press
+	target.dispatchEvent(new PointerEvent('pointerdown', pointerOpts))
+	target.dispatchEvent(new MouseEvent('mousedown', mouseOpts))
+
+	// Focus the original element (nearest focusable ancestor), matching browsers.
+	element.focus({ preventScroll: true })
+
+	// Release
+	target.dispatchEvent(new PointerEvent('pointerup', pointerOpts))
+	target.dispatchEvent(new MouseEvent('mouseup', mouseOpts))
+
+	// Click — activation behavior (navigation, form submit, etc.) triggers
+	// via bubbling from target up to the interactive ancestor.
+	target.click()
+
+	await waitFor(0.2)
+}
+
+export async function inputTextElement(element, text) {
+	const isContentEditable = element.isContentEditable
+	if (!isInputElement(element) && !isTextAreaElement(element) && !isContentEditable) {
+		throw new Error('Element is not an input, textarea, or contenteditable')
+	}
+
+	await clickElement(element)
+
+	if (isContentEditable) {
+		// Plan A: synthetic InputEvents (React contenteditable, Quill).
+		if (
+			element.dispatchEvent(
+				new InputEvent('beforeinput', { bubbles: true, cancelable: true, inputType: 'deleteContent' })
+			)
+		) {
+			element.innerText = ''
+			element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }))
+		}
+
+		if (
+			element.dispatchEvent(
+				new InputEvent('beforeinput', {
+					bubbles: true,
+					cancelable: true,
+					inputType: 'insertText',
+					data: text,
+				})
+			)
+		) {
+			element.innerText = text
+			element.dispatchEvent(
+				new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })
+			)
+		}
+
+		// Verify Plan A actually inserted the text.
+		const planASucceeded = element.innerText.trim() === text.trim()
+
+		if (!planASucceeded) {
+			// Plan B: execCommand fallback (deprecated but widely supported;
+			// integrates with the browser undo stack, handled natively by
+			// most rich-text editors: Quill, Slate.js, react-contenteditable).
+			element.focus()
+			const doc = element.ownerDocument
+			const selection = (doc.defaultView || window).getSelection()
+			const range = doc.createRange()
+			range.selectNodeContents(element)
+			selection?.removeAllRanges()
+			selection?.addRange(range)
+
+			doc.execCommand('delete', false)
+			doc.execCommand('insertText', false, text)
+		}
+
+		element.dispatchEvent(new Event('change', { bubbles: true }))
+		element.blur()
+	} else {
+		// Native prototype setter so React controlled components see the change.
+		getNativeValueSetter(element).call(element, text)
+	}
+
+	if (!isContentEditable) {
+		element.dispatchEvent(new Event('input', { bubbles: true }))
+	}
+
+	await waitFor(0.1)
+	blurLastClickedElement()
+}
+
+export async function selectOptionElement(selectElement, optionText) {
+	if (!isSelectElement(selectElement)) {
+		throw new Error('Element is not a select element')
+	}
+
+	const options = Array.from(selectElement.options)
+	const option = options.find((opt) => opt.textContent?.trim() === optionText.trim())
+
+	if (!option) {
+		throw new Error(`Option with text "${optionText}" not found in select element`)
+	}
+
+	selectElement.value = option.value
+	selectElement.dispatchEvent(new Event('change', { bubbles: true }))
+
+	await waitFor(0.1)
+}
+
+export async function scrollIntoViewIfNeeded(element) {
+	if (typeof element.scrollIntoViewIfNeeded === 'function') {
+		element.scrollIntoViewIfNeeded()
+	} else if (typeof element.scrollIntoView === 'function') {
+		element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'nearest' })
+	}
+}
+
+export async function scrollVertically(scroll_amount, element) {
+	// Element-specific scrolling if element is provided
+	if (element) {
+		const targetElement = element
+		let currentElement = targetElement
+		let scrollSuccess = false
+		let scrolledElement = null
+		let scrollDelta = 0
+		let attempts = 0
+		const dy = scroll_amount
+
+		while (currentElement && attempts < 10) {
+			const computedStyle = window.getComputedStyle(currentElement)
+			const hasScrollableY =
+				/(auto|scroll|overlay)/.test(computedStyle.overflowY) ||
+				(computedStyle.scrollbarWidth && computedStyle.scrollbarWidth !== 'auto') ||
+				(computedStyle.scrollbarGutter && computedStyle.scrollbarGutter !== 'auto')
+			const canScrollVertically = currentElement.scrollHeight > currentElement.clientHeight
+
+			if (hasScrollableY && canScrollVertically) {
+				const beforeScroll = currentElement.scrollTop
+				const maxScroll = currentElement.scrollHeight - currentElement.clientHeight
+
+				let scrollAmount = dy / 3
+				if (scrollAmount > 0) {
+					scrollAmount = Math.min(scrollAmount, maxScroll - beforeScroll)
+				} else {
+					scrollAmount = Math.max(scrollAmount, -beforeScroll)
+				}
+
+				currentElement.scrollTop = beforeScroll + scrollAmount
+
+				const afterScroll = currentElement.scrollTop
+				const actualScrollDelta = afterScroll - beforeScroll
+
+				if (Math.abs(actualScrollDelta) > 0.5) {
+					scrollSuccess = true
+					scrolledElement = currentElement
+					scrollDelta = actualScrollDelta
+					break
+				}
+			}
+
+			if (currentElement === document.body || currentElement === document.documentElement) {
+				break
+			}
+			currentElement = currentElement.parentElement
+			attempts++
+		}
+
+		if (scrollSuccess) {
+			return `Scrolled container (${scrolledElement?.tagName}) by ${scrollDelta}px`
+		} else {
+			return `No scrollable container found for element (${targetElement.tagName})`
+		}
+	}
+
+	// Page-level scrolling (default or fallback)
+	const dy = scroll_amount
+	const bigEnough = (el) => el.clientHeight >= window.innerHeight * 0.5
+	const canScroll = (el) =>
+		Boolean(
+			el &&
+				/(auto|scroll|overlay)/.test(getComputedStyle(el).overflowY) &&
+				el.scrollHeight > el.clientHeight &&
+				bigEnough(el)
+		)
+
+	let el = document.activeElement
+	while (el && !canScroll(el) && el !== document.body) el = el.parentElement
+
+	el = canScroll(el)
+		? el
+		: Array.from(document.querySelectorAll('*')).find(canScroll) ||
+			document.scrollingElement ||
+			document.documentElement
+
+	if (el === document.scrollingElement || el === document.documentElement || el === document.body) {
+		const scrollBefore = window.scrollY
+		const scrollMax = document.documentElement.scrollHeight - window.innerHeight
+
+		window.scrollBy(0, dy)
+
+		const scrollAfter = window.scrollY
+		const scrolled = scrollAfter - scrollBefore
+
+		if (Math.abs(scrolled) < 1) {
+			return dy > 0
+				? `⚠️ Already at the bottom of the page, cannot scroll down further.`
+				: `⚠️ Already at the top of the page, cannot scroll up further.`
+		}
+
+		const reachedBottom = dy > 0 && scrollAfter >= scrollMax - 1
+		const reachedTop = dy < 0 && scrollAfter <= 1
+
+		if (reachedBottom) return `✅ Scrolled page by ${scrolled}px. Reached the bottom of the page.`
+		if (reachedTop) return `✅ Scrolled page by ${scrolled}px. Reached the top of the page.`
+		return `✅ Scrolled page by ${scrolled}px.`
+	} else {
+		const warningMsg = `The document is not scrollable. Falling back to container scroll.`
+
+		const scrollBefore = el.scrollTop
+
+		el.scrollBy({ top: dy, behavior: 'smooth' })
+		await waitFor(0.1)
+
+		const scrollMax = el.scrollHeight - el.clientHeight
+		const scrollAfter = el.scrollTop
+		const scrolled = scrollAfter - scrollBefore
+
+		if (Math.abs(scrolled) < 1) {
+			return dy > 0
+				? `⚠️ ${warningMsg} Already at the bottom of container (${el.tagName}), cannot scroll down further.`
+				: `⚠️ ${warningMsg} Already at the top of container (${el.tagName}), cannot scroll up further.`
+		}
+
+		const reachedBottom = dy > 0 && scrollAfter >= scrollMax - 1
+		const reachedTop = dy < 0 && scrollAfter <= 1
+
+		if (reachedBottom)
+			return `✅ ${warningMsg} Scrolled container (${el.tagName}) by ${scrolled}px. Reached the bottom.`
+		if (reachedTop)
+			return `✅ ${warningMsg} Scrolled container (${el.tagName}) by ${scrolled}px. Reached the top.`
+		return `✅ ${warningMsg} Scrolled container (${el.tagName}) by ${scrolled}px.`
+	}
+}
+
+export async function scrollHorizontally(scroll_amount, element) {
+	if (element) {
+		const targetElement = element
+		let currentElement = targetElement
+		let scrollSuccess = false
+		let scrolledElement = null
+		let scrollDelta = 0
+		let attempts = 0
+		const dx = scroll_amount
+
+		while (currentElement && attempts < 10) {
+			const computedStyle = window.getComputedStyle(currentElement)
+			const hasScrollableX =
+				/(auto|scroll|overlay)/.test(computedStyle.overflowX) ||
+				(computedStyle.scrollbarWidth && computedStyle.scrollbarWidth !== 'auto') ||
+				(computedStyle.scrollbarGutter && computedStyle.scrollbarGutter !== 'auto')
+			const canScrollHorizontally = currentElement.scrollWidth > currentElement.clientWidth
+
+			if (hasScrollableX && canScrollHorizontally) {
+				const beforeScroll = currentElement.scrollLeft
+				const maxScroll = currentElement.scrollWidth - currentElement.clientWidth
+
+				let scrollAmount = dx / 3
+				if (scrollAmount > 0) {
+					scrollAmount = Math.min(scrollAmount, maxScroll - beforeScroll)
+				} else {
+					scrollAmount = Math.max(scrollAmount, -beforeScroll)
+				}
+
+				currentElement.scrollLeft = beforeScroll + scrollAmount
+
+				const afterScroll = currentElement.scrollLeft
+				const actualScrollDelta = afterScroll - beforeScroll
+
+				if (Math.abs(actualScrollDelta) > 0.5) {
+					scrollSuccess = true
+					scrolledElement = currentElement
+					scrollDelta = actualScrollDelta
+					break
+				}
+			}
+
+			if (currentElement === document.body || currentElement === document.documentElement) {
+				break
+			}
+			currentElement = currentElement.parentElement
+			attempts++
+		}
+
+		if (scrollSuccess) {
+			return `Scrolled container (${scrolledElement?.tagName}) horizontally by ${scrollDelta}px`
+		} else {
+			return `No horizontally scrollable container found for element (${targetElement.tagName})`
+		}
+	}
+
+	const dx = scroll_amount
+	const bigEnough = (el) => el.clientWidth >= window.innerWidth * 0.5
+	const canScroll = (el) =>
+		Boolean(
+			el &&
+				/(auto|scroll|overlay)/.test(getComputedStyle(el).overflowX) &&
+				el.scrollWidth > el.clientWidth &&
+				bigEnough(el)
+		)
+
+	let el = document.activeElement
+	while (el && !canScroll(el) && el !== document.body) el = el.parentElement
+
+	el = canScroll(el)
+		? el
+		: Array.from(document.querySelectorAll('*')).find(canScroll) ||
+			document.scrollingElement ||
+			document.documentElement
+
+	if (el === document.scrollingElement || el === document.documentElement || el === document.body) {
+		const scrollBefore = window.scrollX
+		const scrollMax = document.documentElement.scrollWidth - window.innerWidth
+
+		window.scrollBy(dx, 0)
+
+		const scrollAfter = window.scrollX
+		const scrolled = scrollAfter - scrollBefore
+
+		if (Math.abs(scrolled) < 1) {
+			return dx > 0
+				? `⚠️ Already at the right edge of the page, cannot scroll right further.`
+				: `⚠️ Already at the left edge of the page, cannot scroll left further.`
+		}
+
+		const reachedRight = dx > 0 && scrollAfter >= scrollMax - 1
+		const reachedLeft = dx < 0 && scrollAfter <= 1
+
+		if (reachedRight) return `✅ Scrolled page by ${scrolled}px. Reached the right edge of the page.`
+		if (reachedLeft) return `✅ Scrolled page by ${scrolled}px. Reached the left edge of the page.`
+		return `✅ Scrolled page horizontally by ${scrolled}px.`
+	} else {
+		const warningMsg = `The document is not scrollable. Falling back to container scroll.`
+
+		const scrollBefore = el.scrollLeft
+
+		el.scrollBy({ left: dx, behavior: 'smooth' })
+		await waitFor(0.1)
+
+		const scrollMax = el.scrollWidth - el.clientWidth
+		const scrollAfter = el.scrollLeft
+		const scrolled = scrollAfter - scrollBefore
+
+		if (Math.abs(scrolled) < 1) {
+			return dx > 0
+				? `⚠️ ${warningMsg} Already at the right edge of container (${el.tagName}), cannot scroll right further.`
+				: `⚠️ ${warningMsg} Already at the left edge of container (${el.tagName}), cannot scroll left further.`
+		}
+
+		const reachedRight = dx > 0 && scrollAfter >= scrollMax - 1
+		const reachedLeft = dx < 0 && scrollAfter <= 1
+
+		if (reachedRight)
+			return `✅ ${warningMsg} Scrolled container (${el.tagName}) by ${scrolled}px. Reached the right edge.`
+		if (reachedLeft)
+			return `✅ ${warningMsg} Scrolled container (${el.tagName}) by ${scrolled}px. Reached the left edge.`
+		return `✅ ${warningMsg} Scrolled container (${el.tagName}) horizontally by ${scrolled}px.`
+	}
+}