@skippr/live-agent-sdk 0.56.0 → 0.58.0

package/dist/esm/lib-exports.js

@@ -1210,9 +1210,17 @@ var SETTLE_TO_CENTER_MS = 420;
 var ACT_CLICK_DELAY_MS = 400;
 var BASIC_ACT_DELAY_MS = 150;
 var TRACK_GLIDE_MS = 90;
+var GESTURE_GLIDE_MS = 500;
+var GESTURE_ACT_DELAY_MS = 200;
 function cursorEntryPoint() {
   return { x: window.innerWidth - 72, y: window.innerHeight - 80 };
 }
+function gesturePoint(direction) {
+  return {
+    x: window.innerWidth / 2,
+    y: direction === "down" ? window.innerHeight - 96 : 96
+  };
+}
 function centerOf(rect) {
   return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
 }
@@ -1295,7 +1303,18 @@ function AgentCursor({
   useLayoutEffect(() => {
     cancelAnimation();
     setClickPulse(0);
-    if (!target?.element.isConnected) {
+    if (target?.kind === "gesture" && target.gesture) {
+      setPhase("done");
+      setRingBox(null);
+      const destination = gesturePoint(target.gesture.direction);
+      startGlide(prevCursorRef.current ?? cursorEntryPoint(), destination, GESTURE_GLIDE_MS, () => {
+        if (target.mode === "act") {
+          after(GESTURE_ACT_DELAY_MS, () => target.onAct?.());
+        }
+      });
+      return cancelAnimation;
+    }
+    if (!target?.element?.isConnected) {
       setPhase("idle");
       setRingBox(null);
       setCursor(null);
@@ -1346,11 +1365,11 @@ function AgentCursor({
     return cancelAnimation;
   }, [target, animated]);
   useEffect6(() => {
-    if (!target || phase === "idle")
+    if (!target || target.kind === "gesture" || phase === "idle")
       return;
     const track = () => {
       trackFrameRef.current = null;
-      if (!target.element.isConnected)
+      if (!target.element?.isConnected)
         return;
       const rect = getRectInTopViewport(target.element);
       setRingBox(rect);
@@ -1424,6 +1443,21 @@ function AgentCursor({
           })
         ]
       }),
+      target.kind === "gesture" && /* @__PURE__ */ jsx("div", {
+        role: "status",
+        className: "skippr:fixed skippr:-translate-x-1/2 skippr:flex skippr:items-center skippr:gap-1.5 skippr:rounded-full skippr:bg-[#2bc0ae] skippr:px-2.5 skippr:py-1 skippr:shadow-[0_4px_12px_rgba(0,0,0,0.12)]",
+        style: {
+          left: `${cursor.x}px`,
+          top: `${cursor.y + 18}px`,
+          zIndex: Z_INDEX,
+          pointerEvents: "none",
+          transition: `left ${glideMs}ms ${GLIDE_EASE}, top ${glideMs}ms ${GLIDE_EASE}`
+        },
+        children: /* @__PURE__ */ jsx("span", {
+          className: "skippr:text-[11px] skippr:font-medium skippr:text-white skippr:whitespace-nowrap",
+          children: target.label
+        })
+      }),
       /* @__PURE__ */ jsx("div", {
         ref: cursorElRef,
         style: {
@@ -1671,6 +1705,24 @@ function isInteractive(element, role) {
 function isBlindRegion(role) {
   return role === "video" || role === "iframe" || role === "canvas" || role === "audio" || role === "embed" || role === "object";
 }
+var SCROLLABLE_CONTAINER_ROLES = new Set([
+  "listbox",
+  "menu",
+  "dialog",
+  "list",
+  "grid",
+  "tree",
+  "group",
+  "region"
+]);
+function isScrollableContainer(element, role) {
+  if (!SCROLLABLE_CONTAINER_ROLES.has(role))
+    return false;
+  if (element.scrollHeight - element.clientHeight <= 1)
+    return false;
+  const overflowY = window.getComputedStyle(element).overflowY;
+  return overflowY === "auto" || overflowY === "scroll" || overflowY === "overlay";
+}
 function getIframeHost(element) {
   const src = element.getAttribute("src");
   if (!src)
@@ -1748,7 +1800,47 @@ function isActionBlocked(element) {
 function escapeAttributeValue(value) {
   return value.replace(/\\/g, "\\\\").replace(/"/g, "\\\"").replace(/[\r\n]+/g, " ");
 }
-function extractAttrs(element, role) {
+var GENERIC_ICON_NAMES = new Set(["icon", "icons", "image", "img", "sprite", "sprites", "asset"]);
+function sanitizeIconName(src) {
+  if (!src || src.startsWith("data:"))
+    return null;
+  const withoutQuery = src.split(/[?#]/)[0] ?? "";
+  const basename = withoutQuery.slice(withoutQuery.lastIndexOf("/") + 1);
+  const core = basename.split(".")[0] ?? "";
+  const cleaned = core.replace(/^(?:icon[-_]?|ic[-_]|svg[-_])/i, "").replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/[-_]+/g, " ").replace(/\s+/g, " ").trim().toLowerCase().replace(/\s*\bicons?$/, "").trim();
+  if (!cleaned)
+    return null;
+  if (/^[0-9a-f]{8,}$/i.test(cleaned))
+    return null;
+  if (/^\d+$/.test(cleaned))
+    return null;
+  if (GENERIC_ICON_NAMES.has(cleaned))
+    return null;
+  if (cleaned.split(" ").length > 3)
+    return null;
+  return cleaned.slice(0, NAME_MAX_CHARS);
+}
+function hasInteractiveAncestorWithin(image, boundary) {
+  let node = image.parentElement;
+  while (node && node !== boundary) {
+    if (isInteractive(node, inferRole(node)))
+      return true;
+    node = node.parentElement;
+  }
+  return false;
+}
+function iconNameHint(element) {
+  if (element.tagName === "IMG") {
+    return sanitizeIconName(element.getAttribute("src") || "");
+  }
+  const image = element.querySelector("img[src]");
+  if (!image)
+    return null;
+  if (hasInteractiveAncestorWithin(image, element))
+    return null;
+  return sanitizeIconName(image.getAttribute("src") || "");
+}
+function extractAttrs(element, role, name) {
   const attributes = [];
   if (role === "link") {
     const href = element.getAttribute("href");
@@ -1832,6 +1924,13 @@ function extractAttrs(element, role) {
     if (/\.gif(\?|$)/i.test(imageSrc))
       attributes.push("animated");
   }
+  if (!name && isInteractive(element, role)) {
+    const hint = iconNameHint(element);
+    if (hint)
+      attributes.push(`iconName="${escapeAttributeValue(hint)}"`);
+  }
+  if (isScrollableContainer(element, role))
+    attributes.push("scrollable");
   const isNativelyDisabled = "disabled" in element && element.disabled === true;
   if (isNativelyDisabled || element.getAttribute("aria-disabled") === "true") {
     attributes.push("disabled");
@@ -1867,7 +1966,7 @@ function shouldSkipSubtree(element) {
 function shouldEmitElement(element, role) {
   if (role === null)
     return false;
-  return isInteractive(element, role) || role === "heading" || role === "image" || role === "label" || isBlindRegion(role);
+  return isInteractive(element, role) || role === "heading" || role === "image" || role === "label" || isBlindRegion(role) || isScrollableContainer(element, role);
 }
 function getSameOriginIframeBody(element) {
   if (element.tagName !== "IFRAME")
@@ -1889,12 +1988,13 @@ function walkAndCollect(element, depth, entries) {
   let childDepth = depth;
   if (isVisibleAndEmittable && role) {
     const ref = ensureStableRef(element);
+    const name = accessibleName(element);
     entries.push({
       depth,
       role,
-      name: accessibleName(element),
+      name,
       ref,
-      attrs: extractAttrs(element, role),
+      attrs: extractAttrs(element, role, name),
       area: rect.width * rect.height
     });
     childDepth = depth + 1;
@@ -2872,6 +2972,9 @@ import { useDataChannel as useDataChannel2 } from "@livekit/components-react/hoo
 import { useCallback as useCallback6, useEffect as useEffect12, useRef as useRef8 } from "react";
 var STEP_MAX_WAIT_MS = 5000;
 var textDecoder2 = new TextDecoder;
+function parseScrollDirection(value) {
+  return value === "down" || value === "up" || value === "top" || value === "bottom" ? value : null;
+}
 function parsePageAction(payload) {
   try {
     const json = JSON.parse(textDecoder2.decode(payload));
@@ -2886,6 +2989,16 @@ function parsePageAction(payload) {
         return typeof json.ref === "string" && typeof json.value === "string" ? { type: "select", ref: json.ref, value: json.value } : null;
       case "scroll":
         return typeof json.ref === "string" ? { type: "scroll", ref: json.ref } : null;
+      case "hover":
+        return typeof json.ref === "string" ? { type: "hover", ref: json.ref } : null;
+      case "scroll_page": {
+        const direction = parseScrollDirection(json.direction);
+        return direction ? { type: "scroll_page", direction } : null;
+      }
+      case "scroll_in": {
+        const direction = parseScrollDirection(json.direction);
+        return typeof json.ref === "string" && direction ? { type: "scroll_in", ref: json.ref, direction } : null;
+      }
       case "navigate":
         return typeof json.url === "string" ? { type: "navigate", url: json.url } : null;
       default:
@@ -2929,6 +3042,12 @@ function performFill(element, value) {
   if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
     element.focus();
     setReactCompatibleValue(element, value);
+    return;
+  }
+  if (element.isContentEditable) {
+    element.focus();
+    element.textContent = value;
+    element.dispatchEvent(new InputEvent("input", { bubbles: true }));
   }
 }
 function performSelect(element, value) {
@@ -2975,6 +3094,87 @@ function performClick(element) {
   } catch {}
   element.click();
 }
+function performHover(element) {
+  const rect = element.getBoundingClientRect();
+  const base = {
+    bubbles: true,
+    cancelable: true,
+    composed: true,
+    clientX: rect.left + rect.width / 2,
+    clientY: rect.top + rect.height / 2
+  };
+  try {
+    element.dispatchEvent(new PointerEvent("pointerover", { ...base, pointerId: 1, isPrimary: true }));
+  } catch {}
+  element.dispatchEvent(new MouseEvent("mouseover", base));
+  try {
+    element.dispatchEvent(new PointerEvent("pointerenter", { ...base, pointerId: 1, isPrimary: true }));
+  } catch {}
+  element.dispatchEvent(new MouseEvent("mouseenter", base));
+}
+var SCROLL_PAGE_FRACTION = 0.85;
+function isScrollableY(element) {
+  if (element.scrollHeight - element.clientHeight <= 1)
+    return false;
+  const overflowY = element.ownerDocument.defaultView?.getComputedStyle(element).overflowY;
+  return overflowY === "auto" || overflowY === "scroll" || overflowY === "overlay";
+}
+function nearestScrollableContainer(element) {
+  let node = element;
+  while (node && node !== document.body && node !== document.documentElement) {
+    if (isScrollableY(node))
+      return node;
+    node = node.parentElement;
+  }
+  return document.documentElement;
+}
+var PAGE_SCROLLER_MAX_DEPTH = 3;
+function collectShallow(element, depth, out) {
+  for (const child of Array.from(element.children)) {
+    if (child instanceof HTMLElement)
+      out.push(child);
+    if (depth > 1)
+      collectShallow(child, depth - 1, out);
+  }
+}
+function pickPageScroller() {
+  const documentScroller = document.scrollingElement;
+  if (documentScroller && documentScroller.scrollHeight - documentScroller.clientHeight > 1) {
+    return documentScroller;
+  }
+  const candidates = [];
+  collectShallow(document.body, PAGE_SCROLLER_MAX_DEPTH, candidates);
+  let largest = null;
+  let largestArea = 0;
+  for (const element of candidates) {
+    if (!isScrollableY(element))
+      continue;
+    const rect = element.getBoundingClientRect();
+    if (rect.bottom <= 0 || rect.top >= window.innerHeight)
+      continue;
+    const area = rect.width * rect.height;
+    if (area > largestArea) {
+      largestArea = area;
+      largest = element;
+    }
+  }
+  return largest ?? documentScroller ?? document.documentElement;
+}
+function gestureFor(direction) {
+  return direction === "up" || direction === "top" ? "up" : "down";
+}
+function performScroll(container, direction) {
+  if (direction === "top") {
+    container.scrollTo({ top: 0, behavior: "smooth" });
+    return;
+  }
+  if (direction === "bottom") {
+    container.scrollTo({ top: container.scrollHeight, behavior: "smooth" });
+    return;
+  }
+  const delta = container.clientHeight * SCROLL_PAGE_FRACTION * (direction === "down" ? 1 : -1);
+  container.scrollBy({ top: delta, behavior: "smooth" });
+}
 function performAction(action, element) {
   switch (action.type) {
     case "click": {
@@ -2994,6 +3194,12 @@ function performAction(action, element) {
     case "scroll":
       element.scrollIntoView({ behavior: "smooth", block: "center" });
       return;
+    case "hover":
+      performHover(element);
+      return;
+    case "scroll_in":
+      performScroll(nearestScrollableContainer(element), action.direction);
+      return;
   }
 }
 function labelFor(action) {
@@ -3005,7 +3211,11 @@ function labelFor(action) {
     case "select":
       return "Selecting…";
     case "scroll":
+    case "scroll_page":
+    case "scroll_in":
       return "Scrolling…";
+    case "hover":
+      return "Hovering…";
     case "navigate":
       return "Navigating…";
   }
@@ -3023,6 +3233,30 @@ function PageActionHandler() {
         resolve();
         return;
       }
+      if (action.type === "scroll_page") {
+        let scrolled = false;
+        const finishScroll = () => {
+          if (scrolled)
+            return;
+          scrolled = true;
+          resolve();
+        };
+        const fallback2 = setTimeout(finishScroll, STEP_MAX_WAIT_MS);
+        setCursorTarget("act", {
+          kind: "gesture",
+          gesture: { direction: gestureFor(action.direction) },
+          label: labelFor(action),
+          mode: "act",
+          onAct: () => {
+            clearTimeout(fallback2);
+            if (mountedRef.current) {
+              runAgentAction(() => performScroll(pickPageScroller(), action.direction));
+            }
+            finishScroll();
+          }
+        });
+        return;
+      }
       const element = findElementByRef(action.ref);
       if (!element?.isConnected || !isVisible2(element) || isActionBlocked(element)) {
         resolve();