@skippr/live-agent-sdk 0.55.0 → 0.57.0

package/dist/esm/lib-exports.js

@@ -1210,9 +1210,17 @@ var SETTLE_TO_CENTER_MS = 420;
 var ACT_CLICK_DELAY_MS = 400;
 var BASIC_ACT_DELAY_MS = 150;
 var TRACK_GLIDE_MS = 90;
+var GESTURE_GLIDE_MS = 500;
+var GESTURE_ACT_DELAY_MS = 200;
 function cursorEntryPoint() {
   return { x: window.innerWidth - 72, y: window.innerHeight - 80 };
 }
+function gesturePoint(direction) {
+  return {
+    x: window.innerWidth / 2,
+    y: direction === "down" ? window.innerHeight - 96 : 96
+  };
+}
 function centerOf(rect) {
   return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
 }
@@ -1295,7 +1303,18 @@ function AgentCursor({
   useLayoutEffect(() => {
     cancelAnimation();
     setClickPulse(0);
-    if (!target?.element.isConnected) {
+    if (target?.kind === "gesture" && target.gesture) {
+      setPhase("done");
+      setRingBox(null);
+      const destination = gesturePoint(target.gesture.direction);
+      startGlide(prevCursorRef.current ?? cursorEntryPoint(), destination, GESTURE_GLIDE_MS, () => {
+        if (target.mode === "act") {
+          after(GESTURE_ACT_DELAY_MS, () => target.onAct?.());
+        }
+      });
+      return cancelAnimation;
+    }
+    if (!target?.element?.isConnected) {
       setPhase("idle");
       setRingBox(null);
       setCursor(null);
@@ -1346,11 +1365,11 @@ function AgentCursor({
     return cancelAnimation;
   }, [target, animated]);
   useEffect6(() => {
-    if (!target || phase === "idle")
+    if (!target || target.kind === "gesture" || phase === "idle")
       return;
     const track = () => {
       trackFrameRef.current = null;
-      if (!target.element.isConnected)
+      if (!target.element?.isConnected)
         return;
       const rect = getRectInTopViewport(target.element);
       setRingBox(rect);
@@ -1424,6 +1443,21 @@ function AgentCursor({
           })
         ]
       }),
+      target.kind === "gesture" && /* @__PURE__ */ jsx("div", {
+        role: "status",
+        className: "skippr:fixed skippr:-translate-x-1/2 skippr:flex skippr:items-center skippr:gap-1.5 skippr:rounded-full skippr:bg-[#2bc0ae] skippr:px-2.5 skippr:py-1 skippr:shadow-[0_4px_12px_rgba(0,0,0,0.12)]",
+        style: {
+          left: `${cursor.x}px`,
+          top: `${cursor.y + 18}px`,
+          zIndex: Z_INDEX,
+          pointerEvents: "none",
+          transition: `left ${glideMs}ms ${GLIDE_EASE}, top ${glideMs}ms ${GLIDE_EASE}`
+        },
+        children: /* @__PURE__ */ jsx("span", {
+          className: "skippr:text-[11px] skippr:font-medium skippr:text-white skippr:whitespace-nowrap",
+          children: target.label
+        })
+      }),
       /* @__PURE__ */ jsx("div", {
         ref: cursorElRef,
         style: {
@@ -1671,6 +1705,24 @@ function isInteractive(element, role) {
 function isBlindRegion(role) {
   return role === "video" || role === "iframe" || role === "canvas" || role === "audio" || role === "embed" || role === "object";
 }
+var SCROLLABLE_CONTAINER_ROLES = new Set([
+  "listbox",
+  "menu",
+  "dialog",
+  "list",
+  "grid",
+  "tree",
+  "group",
+  "region"
+]);
+function isScrollableContainer(element, role) {
+  if (!SCROLLABLE_CONTAINER_ROLES.has(role))
+    return false;
+  if (element.scrollHeight - element.clientHeight <= 1)
+    return false;
+  const overflowY = window.getComputedStyle(element).overflowY;
+  return overflowY === "auto" || overflowY === "scroll" || overflowY === "overlay";
+}
 function getIframeHost(element) {
   const src = element.getAttribute("src");
   if (!src)
@@ -1832,6 +1884,8 @@ function extractAttrs(element, role) {
     if (/\.gif(\?|$)/i.test(imageSrc))
       attributes.push("animated");
   }
+  if (isScrollableContainer(element, role))
+    attributes.push("scrollable");
   const isNativelyDisabled = "disabled" in element && element.disabled === true;
   if (isNativelyDisabled || element.getAttribute("aria-disabled") === "true") {
     attributes.push("disabled");
@@ -1867,7 +1921,7 @@ function shouldSkipSubtree(element) {
 function shouldEmitElement(element, role) {
   if (role === null)
     return false;
-  return isInteractive(element, role) || role === "heading" || role === "image" || role === "label" || isBlindRegion(role);
+  return isInteractive(element, role) || role === "heading" || role === "image" || role === "label" || isBlindRegion(role) || isScrollableContainer(element, role);
 }
 function getSameOriginIframeBody(element) {
   if (element.tagName !== "IFRAME")
@@ -2160,8 +2214,9 @@ async function snapToCanvas(element, options = {}) {
 }
 
 // src/components/DomCapture.tsx
-var SNAPSHOT_INTERVAL_MS = 2000;
+var SNAPSHOT_INTERVAL_MS = 3000;
 var A11Y_PUBLISH_INTERVAL_MS = 2000;
+var FIRST_SNAPSHOT_DELAY_MS = 400;
 var CAPTURE_PRESET = ScreenSharePresets.h1080fps30;
 var CAPTURE_FPS = CAPTURE_PRESET.encoding.maxFramerate ?? 30;
 var CAPTURE_BITRATE = 4000000;
@@ -2204,7 +2259,7 @@ async function paintViewportSnapshot(canvas, ctx) {
     filter: shouldIncludeInSnapshot,
     filterMode: "remove",
     backgroundColor: "#ffffff",
-    fast: true,
+    fast: false,
     dpr
   });
   const sourceX = window.scrollX * dpr;
@@ -2346,14 +2401,23 @@ function DomCapture() {
         tickSnapshot();
       }
     });
-    tickSnapshot();
+    let snapshotTimer = null;
+    const scheduleNextSnapshot = (delay) => {
+      if (cancelled)
+        return;
+      snapshotTimer = setTimeout(async () => {
+        await tickSnapshot();
+        scheduleNextSnapshot(SNAPSHOT_INTERVAL_MS);
+      }, delay);
+    };
+    scheduleNextSnapshot(FIRST_SNAPSHOT_DELAY_MS);
     tickA11yPublish();
-    const snapshotTimer = setInterval(tickSnapshot, SNAPSHOT_INTERVAL_MS);
     const a11yPublishTimer = setInterval(tickA11yPublish, A11Y_PUBLISH_INTERVAL_MS);
     return () => {
       cancelled = true;
       didStartRef.current = false;
-      clearInterval(snapshotTimer);
+      if (snapshotTimer)
+        clearTimeout(snapshotTimer);
       clearInterval(a11yPublishTimer);
       cleanupDomEventListeners();
       unpublishAndStopTrack(localParticipant, videoTrack);
@@ -2862,6 +2926,9 @@ import { useDataChannel as useDataChannel2 } from "@livekit/components-react/hoo
 import { useCallback as useCallback6, useEffect as useEffect12, useRef as useRef8 } from "react";
 var STEP_MAX_WAIT_MS = 5000;
 var textDecoder2 = new TextDecoder;
+function parseScrollDirection(value) {
+  return value === "down" || value === "up" || value === "top" || value === "bottom" ? value : null;
+}
 function parsePageAction(payload) {
   try {
     const json = JSON.parse(textDecoder2.decode(payload));
@@ -2876,6 +2943,16 @@ function parsePageAction(payload) {
         return typeof json.ref === "string" && typeof json.value === "string" ? { type: "select", ref: json.ref, value: json.value } : null;
       case "scroll":
         return typeof json.ref === "string" ? { type: "scroll", ref: json.ref } : null;
+      case "hover":
+        return typeof json.ref === "string" ? { type: "hover", ref: json.ref } : null;
+      case "scroll_page": {
+        const direction = parseScrollDirection(json.direction);
+        return direction ? { type: "scroll_page", direction } : null;
+      }
+      case "scroll_in": {
+        const direction = parseScrollDirection(json.direction);
+        return typeof json.ref === "string" && direction ? { type: "scroll_in", ref: json.ref, direction } : null;
+      }
       case "navigate":
         return typeof json.url === "string" ? { type: "navigate", url: json.url } : null;
       default:
@@ -2919,6 +2996,12 @@ function performFill(element, value) {
   if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
     element.focus();
     setReactCompatibleValue(element, value);
+    return;
+  }
+  if (element.isContentEditable) {
+    element.focus();
+    element.textContent = value;
+    element.dispatchEvent(new InputEvent("input", { bubbles: true }));
   }
 }
 function performSelect(element, value) {
@@ -2965,6 +3048,87 @@ function performClick(element) {
   } catch {}
   element.click();
 }
+function performHover(element) {
+  const rect = element.getBoundingClientRect();
+  const base = {
+    bubbles: true,
+    cancelable: true,
+    composed: true,
+    clientX: rect.left + rect.width / 2,
+    clientY: rect.top + rect.height / 2
+  };
+  try {
+    element.dispatchEvent(new PointerEvent("pointerover", { ...base, pointerId: 1, isPrimary: true }));
+  } catch {}
+  element.dispatchEvent(new MouseEvent("mouseover", base));
+  try {
+    element.dispatchEvent(new PointerEvent("pointerenter", { ...base, pointerId: 1, isPrimary: true }));
+  } catch {}
+  element.dispatchEvent(new MouseEvent("mouseenter", base));
+}
+var SCROLL_PAGE_FRACTION = 0.85;
+function isScrollableY(element) {
+  if (element.scrollHeight - element.clientHeight <= 1)
+    return false;
+  const overflowY = element.ownerDocument.defaultView?.getComputedStyle(element).overflowY;
+  return overflowY === "auto" || overflowY === "scroll" || overflowY === "overlay";
+}
+function nearestScrollableContainer(element) {
+  let node = element;
+  while (node && node !== document.body && node !== document.documentElement) {
+    if (isScrollableY(node))
+      return node;
+    node = node.parentElement;
+  }
+  return document.documentElement;
+}
+var PAGE_SCROLLER_MAX_DEPTH = 3;
+function collectShallow(element, depth, out) {
+  for (const child of Array.from(element.children)) {
+    if (child instanceof HTMLElement)
+      out.push(child);
+    if (depth > 1)
+      collectShallow(child, depth - 1, out);
+  }
+}
+function pickPageScroller() {
+  const documentScroller = document.scrollingElement;
+  if (documentScroller && documentScroller.scrollHeight - documentScroller.clientHeight > 1) {
+    return documentScroller;
+  }
+  const candidates = [];
+  collectShallow(document.body, PAGE_SCROLLER_MAX_DEPTH, candidates);
+  let largest = null;
+  let largestArea = 0;
+  for (const element of candidates) {
+    if (!isScrollableY(element))
+      continue;
+    const rect = element.getBoundingClientRect();
+    if (rect.bottom <= 0 || rect.top >= window.innerHeight)
+      continue;
+    const area = rect.width * rect.height;
+    if (area > largestArea) {
+      largestArea = area;
+      largest = element;
+    }
+  }
+  return largest ?? documentScroller ?? document.documentElement;
+}
+function gestureFor(direction) {
+  return direction === "up" || direction === "top" ? "up" : "down";
+}
+function performScroll(container, direction) {
+  if (direction === "top") {
+    container.scrollTo({ top: 0, behavior: "smooth" });
+    return;
+  }
+  if (direction === "bottom") {
+    container.scrollTo({ top: container.scrollHeight, behavior: "smooth" });
+    return;
+  }
+  const delta = container.clientHeight * SCROLL_PAGE_FRACTION * (direction === "down" ? 1 : -1);
+  container.scrollBy({ top: delta, behavior: "smooth" });
+}
 function performAction(action, element) {
   switch (action.type) {
     case "click": {
@@ -2984,6 +3148,12 @@ function performAction(action, element) {
     case "scroll":
       element.scrollIntoView({ behavior: "smooth", block: "center" });
       return;
+    case "hover":
+      performHover(element);
+      return;
+    case "scroll_in":
+      performScroll(nearestScrollableContainer(element), action.direction);
+      return;
   }
 }
 function labelFor(action) {
@@ -2995,7 +3165,11 @@ function labelFor(action) {
     case "select":
       return "Selecting…";
     case "scroll":
+    case "scroll_page":
+    case "scroll_in":
       return "Scrolling…";
+    case "hover":
+      return "Hovering…";
     case "navigate":
       return "Navigating…";
   }
@@ -3013,6 +3187,30 @@ function PageActionHandler() {
         resolve();
         return;
       }
+      if (action.type === "scroll_page") {
+        let scrolled = false;
+        const finishScroll = () => {
+          if (scrolled)
+            return;
+          scrolled = true;
+          resolve();
+        };
+        const fallback2 = setTimeout(finishScroll, STEP_MAX_WAIT_MS);
+        setCursorTarget("act", {
+          kind: "gesture",
+          gesture: { direction: gestureFor(action.direction) },
+          label: labelFor(action),
+          mode: "act",
+          onAct: () => {
+            clearTimeout(fallback2);
+            if (mountedRef.current) {
+              runAgentAction(() => performScroll(pickPageScroller(), action.direction));
+            }
+            finishScroll();
+          }
+        });
+        return;
+      }
       const element = findElementByRef(action.ref);
       if (!element?.isConnected || !isVisible2(element) || isActionBlocked(element)) {
         resolve();