@xpert-ai/chatkit-host-automation 0.3.1 → 0.3.2

package/README.md

@@ -23,3 +23,12 @@ The first version exposes a fixed tool allow-list:
 - `host_page_select`
 - `host_page_scroll`
 - `host_page_navigate`
+- `host_page_hover`
+- `host_page_focus`
+- `host_page_pointer`
+- `host_page_screenshot`
+- `host_page_wait_for`
+
+Plain web hosts use the DOM fallback executor. The Chrome browser extension can
+upgrade the same client tools through the Chrome DevTools Protocol for richer
+snapshots and browser-level input.

package/dist/index.d.ts

@@ -2,7 +2,7 @@ import { ClientToolMessageInput } from '../../chatkit/src/index.ts';
 
 export declare function createHostPageAutomationClientToolHandler(options?: HostPageAutomationOptions): HostPageAutomationClientToolHandler;
 
-export declare const HOST_PAGE_AUTOMATION_TOOL_NAMES: readonly ["host_page_snapshot", "host_page_click", "host_page_fill", "host_page_press", "host_page_select", "host_page_scroll", "host_page_navigate"];
+export declare const HOST_PAGE_AUTOMATION_TOOL_NAMES: readonly ["host_page_snapshot", "host_page_click", "host_page_fill", "host_page_press", "host_page_select", "host_page_scroll", "host_page_navigate", "host_page_hover", "host_page_focus", "host_page_pointer", "host_page_screenshot", "host_page_wait_for"];
 
 export declare type HostPageAutomationClientToolCall = {
     name: string;
@@ -19,17 +19,38 @@ export declare type HostPageAutomationElementSnapshot = {
     role?: string;
     name?: string;
     text?: string;
+    nearbyText?: string[];
+    testId?: string;
     value?: string;
     placeholder?: string;
     selector?: string;
     disabled?: boolean;
+    enabled?: boolean;
     checked?: boolean;
+    visible?: boolean;
+    actionable?: boolean;
     rect: {
         x: number;
         y: number;
         width: number;
         height: number;
     };
+    center?: {
+        x: number;
+        y: number;
+    };
+    hitTarget?: {
+        tag: string;
+        role?: string;
+        name?: string;
+        selector?: string;
+    };
+    hitStack?: Array<{
+        tag: string;
+        role?: string;
+        name?: string;
+        selector?: string;
+    }>;
 };
 
 export declare class HostPageAutomationExecutor {
@@ -40,7 +61,11 @@ export declare class HostPageAutomationExecutor {
     execute(name: HostPageAutomationToolName, params?: Record<string, unknown>): Promise<unknown>;
     snapshot(): HostPageSnapshot;
     private getRoot;
+    private getNavigationState;
+    private getFrameState;
     private snapshotElement;
+    private snapshotElementWithoutRef;
+    private createElementSnapshot;
     private resolveElement;
     private click;
     private fill;
@@ -48,6 +73,12 @@ export declare class HostPageAutomationExecutor {
     private select;
     private scroll;
     private navigate;
+    private hover;
+    private focus;
+    private pointer;
+    private screenshot;
+    private waitFor;
+    private tryResolveElement;
     private describeElement;
 }
 
@@ -62,14 +93,55 @@ export declare type HostPageAutomationToolName = (typeof HOST_PAGE_AUTOMATION_TO
 export declare type HostPageSnapshot = {
     url: string;
     title: string;
+    capabilities?: {
+        cdp?: boolean;
+        realInput?: boolean;
+        screenshot?: boolean;
+        accessibility?: boolean;
+        networkState?: boolean;
+    };
     viewport: {
         width: number;
         height: number;
+        devicePixelRatio?: number;
     };
     scroll: {
         x: number;
         y: number;
     };
+    page?: {
+        readyState?: string;
+        visibilityState?: string;
+        focusedElement?: HostPageAutomationElementSnapshot;
+        selection?: string;
+    };
+    navigation?: {
+        type?: string;
+        duration?: number;
+        domContentLoaded?: number;
+        loadEventEnd?: number;
+    };
+    frames?: Array<{
+        url?: string;
+        title?: string;
+        sameOrigin: boolean;
+        rect?: {
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        };
+    }>;
+    accessibility?: Array<{
+        ref?: string;
+        role?: string;
+        name?: string;
+        value?: string;
+        disabled?: boolean;
+        checked?: boolean;
+        expanded?: boolean;
+        focused?: boolean;
+    }>;
     elements: HostPageAutomationElementSnapshot[];
 };
 

package/dist/index.js

@@ -1,14 +1,19 @@
-const S = [
+const z = [
   "host_page_snapshot",
   "host_page_click",
   "host_page_fill",
   "host_page_press",
   "host_page_select",
   "host_page_scroll",
-  "host_page_navigate"
-], H = new Set(
-  S
-), k = [
+  "host_page_navigate",
+  "host_page_hover",
+  "host_page_focus",
+  "host_page_pointer",
+  "host_page_screenshot",
+  "host_page_wait_for"
+], G = new Set(
+  z
+), K = [
   "a[href]",
   "button",
   "input",
@@ -20,52 +25,64 @@ const S = [
   '[contenteditable=""]',
   '[contenteditable="true"]',
   "[aria-label]",
-  "[data-testid]"
-].join(","), x = 100;
-function N(t) {
-  return H.has(t);
+  "[data-testid]",
+  "[data-test-id]",
+  "[data-qa]"
+].join(","), D = 100, J = 1e4, Q = 6e4;
+function Z(t) {
+  return G.has(t);
 }
-function f(t) {
+function g(t) {
   return t instanceof Document ? t : t.ownerDocument;
 }
-function p(t) {
-  return f(t).defaultView ?? window;
+function N(t) {
+  return g(t).defaultView ?? window;
 }
-function u(t) {
+function h(t) {
   return typeof t == "object" && t !== null && !Array.isArray(t) ? t : {};
 }
-function g(t, e) {
+function R(t, e) {
   if (typeof t != "string" || t.trim() === "")
     throw new Error(`${e} must be a non-empty string.`);
   return t;
 }
-function _(t) {
+function p(t) {
   return typeof t == "string" && t.trim() ? t : void 0;
 }
-function v(t, e) {
+function tt(t) {
+  return typeof t == "number" && Number.isInteger(t) ? t : void 0;
+}
+function M(t, e) {
   if (typeof t != "number" || !Number.isFinite(t))
     throw new Error(`${e} must be a finite number.`);
   return t;
 }
-function h(t) {
+function T(t) {
   return typeof t == "number" && Number.isFinite(t) ? t : void 0;
 }
-function R(t) {
+function F(t, e) {
+  return typeof t == "string" && e.includes(t) ? t : void 0;
+}
+function et(t) {
   if (typeof t == "string")
     return [t];
   if (Array.isArray(t) && t.every((e) => typeof e == "string"))
     return t;
   throw new Error("value or values must be a string or an array of strings.");
 }
-function y(t) {
+function v(t) {
   const e = (t.textContent ?? "").replace(/\s+/g, " ").trim();
   return e ? e.slice(0, 160) : void 0;
 }
-function C(t) {
+function V(t) {
+  const e = Array.from(t.childNodes).filter((n) => n.nodeType === Node.TEXT_NODE).map((n) => n.textContent ?? "").join(" ").replace(/\s+/g, " ").trim();
+  return e ? e.slice(0, 160) : void 0;
+}
+function nt(t) {
   if (t instanceof HTMLInputElement || t instanceof HTMLTextAreaElement || t instanceof HTMLSelectElement)
     return t.value;
 }
-function L(t) {
+function E(t) {
   const e = t.getAttribute("aria-label");
   if (e != null && e.trim())
     return e.trim();
@@ -73,13 +90,54 @@ function L(t) {
   if (n != null && n.trim())
     return n.trim();
   if (t instanceof HTMLInputElement) {
-    const r = Array.from(t.labels ?? []).map((o) => y(o)).filter(Boolean);
+    const r = Array.from(t.labels ?? []).map((c) => v(c)).filter(Boolean);
     if (r.length > 0)
       return r.join(" ").slice(0, 160);
   }
-  return y(t);
+  const o = X(t)[0];
+  return o || v(t);
 }
-function A(t) {
+function ot(t) {
+  if (!(t instanceof HTMLElement || t instanceof SVGElement) || !m(t) || t.matches(
+    'input,textarea,select,button,a[href],[role="button"],[role="link"]'
+  ))
+    return !1;
+  const e = V(t) ?? v(t);
+  return !!(e && e.length <= 120);
+}
+function X(t) {
+  const e = t.ownerDocument, n = e.body;
+  if (!n)
+    return [];
+  const o = t.getBoundingClientRect();
+  if (o.width <= 0 || o.height <= 0)
+    return [];
+  const r = o.top + o.height / 2, c = o.left + o.width / 2, s = [], i = e.createTreeWalker(n, NodeFilter.SHOW_ELEMENT);
+  let a = i.nextNode();
+  for (; a; ) {
+    const l = a;
+    if (l !== t && ot(l)) {
+      const u = l.getBoundingClientRect(), f = V(l) ?? v(l);
+      if (f) {
+        const b = u.top + u.height / 2, w = u.left + u.width / 2, y = u.right <= o.left + 12 && Math.abs(b - r) <= Math.max(28, o.height * 1.25), S = u.bottom <= o.top + 8 && o.top - u.bottom <= 80 && w >= o.left - 80 && w <= o.right + 80;
+        if (y || S) {
+          const j = y ? o.left - u.right + Math.abs(b - r) : o.top - u.bottom + Math.abs(w - c);
+          s.push({
+            text: f,
+            score: j + (y ? 0 : 100)
+          });
+        }
+      }
+    }
+    a = i.nextNode();
+  }
+  const d = /* @__PURE__ */ new Set();
+  return s.sort((l, u) => l.score - u.score).map((l) => l.text).filter((l) => d.has(l) ? !1 : (d.add(l), !0)).slice(0, 4);
+}
+function rt(t) {
+  return t.getAttribute("data-testid") ?? t.getAttribute("data-test-id") ?? t.getAttribute("data-qa") ?? void 0;
+}
+function x(t) {
   const e = t.getAttribute("role");
   if (e != null && e.trim())
     return e.trim();
@@ -103,94 +161,221 @@ function A(t) {
         return "textbox";
     }
 }
-function O(t) {
+function q(t) {
   return t instanceof HTMLButtonElement || t instanceof HTMLInputElement || t instanceof HTMLSelectElement || t instanceof HTMLTextAreaElement ? t.disabled || void 0 : t.getAttribute("aria-disabled") === "true" || void 0;
 }
-function I(t) {
+function it(t) {
   return t instanceof HTMLInputElement && (t.type === "checkbox" || t.type === "radio") ? t.checked : void 0;
 }
-function D(t) {
+function st(t) {
+  const e = k(t);
+  if (!(e.width <= 0 || e.height <= 0))
+    return {
+      x: e.x + e.width / 2,
+      y: e.y + e.height / 2
+    };
+}
+function $(t) {
+  return {
+    tag: t.tagName.toLowerCase(),
+    role: x(t),
+    name: E(t),
+    selector: P(t)
+  };
+}
+function m(t) {
   if (!(t instanceof HTMLElement || t instanceof SVGElement) || t instanceof HTMLElement && t.hidden)
     return !1;
   const e = t.ownerDocument.defaultView, n = e == null ? void 0 : e.getComputedStyle(t);
   return !(n && (n.display === "none" || n.visibility === "hidden" || n.opacity === "0"));
 }
-function $(t) {
-  return t.matches(k) && D(t);
+function U(t) {
+  return t.matches(K) && m(t);
 }
-function P(t) {
-  var s, i;
+function ct(t) {
+  return q(t) !== !0;
+}
+function at(t) {
+  var n;
+  if (U(t))
+    return !0;
+  if (!(t instanceof HTMLElement))
+    return !1;
+  const e = (n = t.ownerDocument.defaultView) == null ? void 0 : n.getComputedStyle(t);
+  return ((e == null ? void 0 : e.cursor) === "pointer" || !!t.onclick) && m(t);
+}
+function k(t) {
+  var c, s;
   const e = t.getBoundingClientRect();
-  let n = e.left, r = e.top, o = null;
+  let n = e.left, o = e.top, r = null;
   try {
-    o = ((s = t.ownerDocument.defaultView) == null ? void 0 : s.frameElement) ?? null;
+    r = ((c = t.ownerDocument.defaultView) == null ? void 0 : c.frameElement) ?? null;
   } catch {
-    o = null;
+    r = null;
   }
-  for (; o; ) {
-    const c = o.getBoundingClientRect();
-    n += c.left, r += c.top;
+  for (; r; ) {
+    const i = r.getBoundingClientRect();
+    n += i.left, o += i.top;
     try {
-      o = ((i = o.ownerDocument.defaultView) == null ? void 0 : i.frameElement) ?? null;
+      r = ((s = r.ownerDocument.defaultView) == null ? void 0 : s.frameElement) ?? null;
     } catch {
-      o = null;
+      r = null;
     }
   }
   return {
     x: n,
-    y: r,
+    y: o,
     width: e.width,
     height: e.height
   };
 }
-function M(t) {
+function P(t) {
   const e = (o) => typeof CSS < "u" && typeof CSS.escape == "function" ? CSS.escape(o) : o.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
   if (t.id)
     return `#${e(t.id)}`;
-  const n = t.getAttribute("data-testid");
-  if (n)
-    return `[data-testid="${e(n)}"]`;
-  const r = t.getAttribute("name");
-  return r ? `${t.tagName.toLowerCase()}[name="${e(
-    r
+  for (const o of ["data-testid", "data-test-id", "data-qa"]) {
+    const r = t.getAttribute(o);
+    if (r)
+      return `[${o}="${e(r)}"]`;
+  }
+  const n = t.getAttribute("name");
+  return n ? `${t.tagName.toLowerCase()}[name="${e(
+    n
   )}"]` : t.tagName.toLowerCase();
 }
-function V(t) {
-  const e = f(t), n = t instanceof Document ? t.body ?? t.documentElement : t;
+function lt(t) {
+  const e = t.getBoundingClientRect();
+  if (!(e.width <= 0 || e.height <= 0))
+    return {
+      x: e.left + e.width / 2,
+      y: e.top + e.height / 2
+    };
+}
+function W(t, e) {
+  if (typeof t.elementsFromPoint == "function")
+    return t.elementsFromPoint(e.x, e.y);
+  if (typeof t.elementFromPoint != "function")
+    return [];
+  const n = t.elementFromPoint(e.x, e.y);
+  return n ? [n] : [];
+}
+function ut(t) {
+  return typeof t.elementsFromPoint == "function" || typeof t.elementFromPoint == "function";
+}
+function C(t, e) {
+  return t === e || t.contains(e);
+}
+function ft(t) {
+  let e = t;
+  for (; e; ) {
+    if (at(e))
+      return e;
+    e = e.parentElement;
+  }
+  return t;
+}
+function B(t) {
+  return (t ?? "").replace(/\([^)]*\)/g, "").replace(/\s+/g, "").trim().toLowerCase();
+}
+function Y(t) {
+  const e = t.getBoundingClientRect();
+  if (e.width <= 0 || e.height <= 0)
+    return;
+  const n = t.ownerDocument;
+  return [
+    { x: e.left + e.width / 2, y: e.top + e.height / 2 },
+    { x: e.left + Math.min(8, e.width / 2), y: e.top + e.height / 2 },
+    { x: e.right - Math.min(8, e.width / 2), y: e.top + e.height / 2 },
+    { x: e.left + e.width / 2, y: e.top + Math.min(8, e.height / 2) },
+    { x: e.left + e.width / 2, y: e.bottom - Math.min(8, e.height / 2) }
+  ].find(
+    (r) => W(n, r).some(
+      (c) => C(t, c) || C(c, t)
+    )
+  );
+}
+function dt(t, e) {
+  var c;
+  const n = x(e), o = B(E(e));
+  return o ? ((c = I(t).filter((s) => s !== e).filter((s) => {
+    const i = x(s), a = B(E(s));
+    return n && i && n !== i ? !1 : a.includes(o) || o.includes(a);
+  }).map((s) => ({
+    candidate: s,
+    point: Y(s),
+    rect: k(s)
+  })).filter(
+    (s) => !!s.point
+  ).sort((s, i) => i.rect.y - s.rect.y)[0]) == null ? void 0 : c.candidate) ?? null : null;
+}
+function L(t) {
+  const e = t.ownerDocument, n = lt(t), o = m(t) && !!n, r = ct(t), c = n ? W(e, n) : [], s = c[0], i = !!(s && (C(t, s) || C(s, t)));
+  return {
+    visible: o,
+    enabled: r,
+    receivesEvents: i,
+    actionable: o && r && i,
+    center: st(t),
+    hitTarget: s,
+    hitStack: c
+  };
+}
+function ht(t, e) {
+  var i, a, d;
+  const n = p(e.testId);
+  if (n) {
+    const l = typeof CSS < "u" && typeof CSS.escape == "function" ? CSS.escape(n) : n.replaceAll("\\", "\\\\").replaceAll('"', '\\"'), u = t.querySelector(
+      `[data-testid="${l}"],[data-test-id="${l}"],[data-qa="${l}"]`
+    );
+    if (u)
+      return u;
+  }
+  const o = (i = p(e.role)) == null ? void 0 : i.toLowerCase(), r = (a = p(e.name)) == null ? void 0 : a.toLowerCase(), c = (d = p(e.text)) == null ? void 0 : d.toLowerCase();
+  return !o && !r && !c ? null : I(t).find((l) => {
+    var w, y, S;
+    const u = (w = x(l)) == null ? void 0 : w.toLowerCase(), f = (y = E(l)) == null ? void 0 : y.toLowerCase(), b = (S = v(l)) == null ? void 0 : S.toLowerCase();
+    return !(o && u !== o || r && !(f != null && f.includes(r)) || c && !(b != null && b.includes(c)));
+  }) ?? null;
+}
+function I(t) {
+  const e = g(t), n = t instanceof Document ? t.body ?? t.documentElement : t;
   if (!n)
     return [];
-  const r = [], o = (s) => {
-    const i = e.createTreeWalker(s, NodeFilter.SHOW_ELEMENT);
-    let c = i.nextNode();
-    for (; c; ) {
-      const l = c;
-      $(l) && r.push(l);
-      const d = l.shadowRoot;
-      if (d && o(d), l instanceof HTMLIFrameElement)
+  const o = [], r = (c) => {
+    const s = e.createTreeWalker(c, NodeFilter.SHOW_ELEMENT);
+    let i = s.nextNode();
+    for (; i; ) {
+      const a = i;
+      U(a) && o.push(a);
+      const d = a.shadowRoot;
+      if (d && r(d), a instanceof HTMLIFrameElement)
         try {
-          l.contentDocument && o(l.contentDocument);
+          a.contentDocument && r(a.contentDocument);
         } catch {
         }
-      if (r.length >= x)
+      if (o.length >= D)
         return;
-      c = i.nextNode();
+      i = s.nextNode();
     }
   };
-  return o(n), r.slice(0, x);
+  return r(n), o.slice(0, D);
 }
-function U(t, e) {
-  var o;
-  const n = Object.getPrototypeOf(t), r = Object.getOwnPropertyDescriptor(n, "value");
-  (o = r == null ? void 0 : r.set) == null || o.call(t, e);
+function gt(t, e) {
+  var r;
+  const n = Object.getPrototypeOf(t), o = Object.getOwnPropertyDescriptor(n, "value");
+  (r = o == null ? void 0 : o.set) == null || r.call(t, e);
 }
-function E(t) {
+function H(t) {
   t.dispatchEvent(new Event("input", { bubbles: !0 })), t.dispatchEvent(new Event("change", { bubbles: !0 }));
 }
-function b(t) {
+function A(t) {
   var e, n;
   (t instanceof HTMLElement || t instanceof SVGElement) && ((e = t.scrollIntoView) == null || e.call(t, { block: "center", inline: "center" })), t instanceof HTMLElement && ((n = t.focus) == null || n.call(t));
 }
-class j {
+function _(t, e) {
+  return typeof PointerEvent == "function" ? new PointerEvent(t, e) : new MouseEvent(t.replace("pointer", "mouse"), e);
+}
+class pt {
   constructor(e = {}) {
     this.options = e, this.refs = /* @__PURE__ */ new Map(), this.nextRef = 1;
   }
@@ -212,187 +397,365 @@ class j {
         return this.scroll(n);
       case "host_page_navigate":
         return this.navigate(n);
+      case "host_page_hover":
+        return this.hover(n);
+      case "host_page_focus":
+        return this.focus(n);
+      case "host_page_pointer":
+        return this.pointer(n);
+      case "host_page_screenshot":
+        return this.screenshot();
+      case "host_page_wait_for":
+        return this.waitFor(n);
     }
   }
   snapshot() {
+    var c, s;
     this.refs.clear(), this.nextRef = 1;
-    const e = this.getRoot(), n = f(e), r = p(e), o = V(e).map(
-      (s) => this.snapshotElement(s)
+    const e = this.getRoot(), n = g(e), o = N(e), r = I(e).map(
+      (i) => this.snapshotElement(i)
     );
     return {
-      url: r.location.href,
+      url: o.location.href,
       title: n.title,
+      capabilities: {
+        cdp: !1,
+        realInput: !1,
+        screenshot: !1,
+        accessibility: !1,
+        networkState: !1
+      },
       viewport: {
-        width: r.innerWidth,
-        height: r.innerHeight
+        width: o.innerWidth,
+        height: o.innerHeight,
+        devicePixelRatio: o.devicePixelRatio
       },
       scroll: {
-        x: r.scrollX,
-        y: r.scrollY
+        x: o.scrollX,
+        y: o.scrollY
+      },
+      page: {
+        readyState: n.readyState,
+        visibilityState: n.visibilityState,
+        focusedElement: n.activeElement && n.activeElement !== n.body ? this.snapshotElementWithoutRef(n.activeElement) : void 0,
+        selection: ((s = (c = n.getSelection) == null ? void 0 : c.call(n)) == null ? void 0 : s.toString()) || void 0
       },
-      elements: o
+      navigation: this.getNavigationState(o),
+      frames: this.getFrameState(n),
+      accessibility: r.map((i) => ({
+        ref: i.ref,
+        role: i.role,
+        name: i.name,
+        value: i.value,
+        disabled: i.disabled,
+        checked: i.checked,
+        focused: n.activeElement === this.refs.get(i.ref)
+      })),
+      elements: r
     };
   }
   getRoot() {
     return this.options.root ?? document;
   }
+  getNavigationState(e) {
+    var o, r, c;
+    const n = (c = (r = (o = e.performance) == null ? void 0 : o.getEntriesByType) == null ? void 0 : r.call(o, "navigation")) == null ? void 0 : c[0];
+    if (!(typeof PerformanceNavigationTiming != "function" || !(n instanceof PerformanceNavigationTiming)))
+      return {
+        type: n.type,
+        duration: n.duration,
+        domContentLoaded: n.domContentLoadedEventEnd - n.startTime,
+        loadEventEnd: n.loadEventEnd - n.startTime
+      };
+  }
+  getFrameState(e) {
+    return Array.from(e.querySelectorAll("iframe")).map((n) => {
+      var r, c;
+      const o = k(n);
+      try {
+        return {
+          url: (r = n.contentWindow) == null ? void 0 : r.location.href,
+          title: (c = n.contentDocument) == null ? void 0 : c.title,
+          sameOrigin: !0,
+          rect: o
+        };
+      } catch {
+        return {
+          url: n.getAttribute("src") ?? void 0,
+          sameOrigin: !1,
+          rect: o
+        };
+      }
+    });
+  }
   snapshotElement(e) {
     const n = `e${this.nextRef}`;
-    this.nextRef += 1, this.refs.set(n, e);
-    const r = (e instanceof HTMLInputElement || e instanceof HTMLTextAreaElement) && e.placeholder || void 0;
+    return this.nextRef += 1, this.refs.set(n, e), this.createElementSnapshot(e, n);
+  }
+  snapshotElementWithoutRef(e) {
+    return this.createElementSnapshot(e, "");
+  }
+  createElementSnapshot(e, n) {
+    const o = (e instanceof HTMLInputElement || e instanceof HTMLTextAreaElement) && e.placeholder || void 0, r = L(e);
     return {
       ref: n,
       tag: e.tagName.toLowerCase(),
-      role: A(e),
-      name: L(e),
-      text: y(e),
-      value: C(e),
-      placeholder: r,
-      selector: M(e),
-      disabled: O(e),
-      checked: I(e),
-      rect: P(e)
+      role: x(e),
+      name: E(e),
+      text: v(e),
+      nearbyText: X(e),
+      testId: rt(e),
+      value: nt(e),
+      placeholder: o,
+      selector: P(e),
+      disabled: q(e),
+      enabled: r.enabled,
+      checked: it(e),
+      visible: r.visible,
+      actionable: r.actionable,
+      rect: k(e),
+      center: r.center,
+      hitTarget: r.hitTarget ? $(r.hitTarget) : void 0,
+      hitStack: r.hitStack.slice(0, 5).map($)
     };
   }
   resolveElement(e) {
-    const n = _(e.ref);
+    const n = p(e.ref) ?? p(e.axRef);
     if (n) {
-      const o = this.refs.get(n);
-      if (!o)
+      const c = this.refs.get(n);
+      if (!c)
         throw new Error(`Unknown element ref: ${n}. Take a new snapshot.`);
-      return o;
+      return c;
     }
-    const r = _(e.selector);
-    if (r) {
-      const o = this.getRoot().querySelector(r);
-      if (!o)
-        throw new Error(`No element matches selector: ${r}.`);
-      return o;
+    const o = p(e.selector);
+    if (o) {
+      const c = this.getRoot().querySelector(o);
+      if (!c)
+        throw new Error(`No element matches selector: ${o}.`);
+      return c;
     }
+    const r = ht(this.getRoot(), e);
+    if (r)
+      return r;
     if (typeof e.x < "u" || typeof e.y < "u") {
-      const o = {
-        x: v(e.x, "x"),
-        y: v(e.y, "y")
-      }, s = f(this.getRoot()).elementFromPoint(
-        o.x,
-        o.y
+      const c = {
+        x: M(e.x, "x"),
+        y: M(e.y, "y")
+      }, s = g(this.getRoot()).elementFromPoint(
+        c.x,
+        c.y
       );
       if (!s)
-        throw new Error(`No element found at (${o.x}, ${o.y}).`);
+        throw new Error(`No element found at (${c.x}, ${c.y}).`);
       return s;
     }
-    throw new Error("Expected one of ref, selector, or x/y.");
+    throw new Error(
+      "Expected one of ref, selector, role/name/text/testId, or x/y."
+    );
   }
   click(e) {
-    const n = this.resolveElement(u(e));
-    return b(n), n instanceof HTMLElement ? n.click() : n.dispatchEvent(new MouseEvent("click", { bubbles: !0 })), { clicked: this.describeElement(n) };
+    const n = h(e), o = this.resolveElement(n), r = L(o), s = (r.actionable ? null : dt(this.getRoot(), o)) ?? ft(o), i = Y(s);
+    if (!i && !r.actionable && ut(o.ownerDocument))
+      throw new Error(
+        `Target "${E(o) ?? o.tagName.toLowerCase()}" is not receiving pointer events. Take a screenshot or use host_page_pointer coordinates.`
+      );
+    return A(s), s instanceof HTMLElement ? s.click() : s.dispatchEvent(new MouseEvent("click", { bubbles: !0 })), {
+      clicked: this.describeElement(s),
+      requested: this.describeElement(o),
+      strategy: s === o ? "dom" : "semantic_visible_fallback",
+      point: i ?? r.center,
+      actionability: {
+        visible: r.visible,
+        enabled: r.enabled,
+        receivesEvents: r.receivesEvents
+      }
+    };
   }
   fill(e) {
-    const n = u(e), r = g(n.value, "value"), o = this.resolveElement(n);
-    if (b(o), o instanceof HTMLInputElement || o instanceof HTMLTextAreaElement)
-      return U(o, r), E(o), { filled: this.describeElement(o), value: r };
-    if (o instanceof HTMLElement && o.isContentEditable)
-      return o.textContent = r, E(o), { filled: this.describeElement(o), value: r };
+    const n = h(e), o = R(n.value, "value"), r = this.resolveElement(n);
+    if (A(r), r instanceof HTMLInputElement || r instanceof HTMLTextAreaElement)
+      return gt(r, o), H(r), { filled: this.describeElement(r), value: o };
+    if (r instanceof HTMLElement && r.isContentEditable)
+      return r.textContent = o, H(r), { filled: this.describeElement(r), value: o };
     throw new Error("Target element cannot be filled.");
   }
   press(e) {
-    const n = u(e), r = g(n.key, "key"), o = n.ref || n.selector || n.x !== void 0 && n.y !== void 0 ? this.resolveElement(n) : f(this.getRoot()).activeElement ?? f(this.getRoot()).body;
-    if (!o)
+    const n = h(e), o = R(n.key, "key"), r = n.ref || n.selector || n.x !== void 0 && n.y !== void 0 ? this.resolveElement(n) : g(this.getRoot()).activeElement ?? g(this.getRoot()).body;
+    if (!r)
       throw new Error("No target element is available for key press.");
-    b(o);
-    const s = { key: r, bubbles: !0, cancelable: !0 };
-    return o.dispatchEvent(new KeyboardEvent("keydown", s)), r.length === 1 && o.dispatchEvent(new KeyboardEvent("keypress", s)), o.dispatchEvent(new KeyboardEvent("keyup", s)), { pressed: r, target: this.describeElement(o) };
+    A(r);
+    const c = { key: o, bubbles: !0, cancelable: !0 };
+    return r.dispatchEvent(new KeyboardEvent("keydown", c)), o.length === 1 && r.dispatchEvent(new KeyboardEvent("keypress", c)), r.dispatchEvent(new KeyboardEvent("keyup", c)), { pressed: o, target: this.describeElement(r) };
   }
   select(e) {
-    const n = u(e), r = R(n.values ?? n.value), o = this.resolveElement(n);
-    if (!(o instanceof HTMLSelectElement))
+    const n = h(e), o = et(n.values ?? n.value), r = this.resolveElement(n);
+    if (!(r instanceof HTMLSelectElement))
       throw new Error("Target element is not a select.");
-    const s = new Set(r);
-    for (const i of Array.from(o.options))
-      i.selected = s.has(i.value);
-    return E(o), {
-      selected: Array.from(o.selectedOptions).map(
-        (i) => i.value
+    const c = new Set(o);
+    for (const s of Array.from(r.options))
+      s.selected = c.has(s.value);
+    return H(r), {
+      selected: Array.from(r.selectedOptions).map(
+        (s) => s.value
       ),
-      target: this.describeElement(o)
+      target: this.describeElement(r)
     };
   }
   scroll(e) {
-    var l, d, w, T;
-    const n = u(e), r = n.x !== void 0 || n.y !== void 0 ? {
-      x: h(n.x) ?? 0,
-      y: h(n.y) ?? 0
-    } : null, o = h(n.deltaX) ?? 0, s = h(n.deltaY) ?? 0, i = this.getRoot(), c = p(i);
-    if (n.ref || n.selector) {
-      const a = this.resolveElement(n);
-      if (!(a instanceof HTMLElement))
+    var a, d, l, u;
+    const n = h(e), o = n.x !== void 0 || n.y !== void 0 ? {
+      x: T(n.x) ?? 0,
+      y: T(n.y) ?? 0
+    } : null, r = T(n.deltaX) ?? 0, c = T(n.deltaY) ?? 0, s = this.getRoot(), i = N(s);
+    if (n.ref || n.selector || n.role || n.name || n.text || n.testId) {
+      const f = this.resolveElement(n);
+      if (!(f instanceof HTMLElement))
         throw new Error("Target element cannot be scrolled.");
-      return r ? (l = a.scrollTo) == null || l.call(a, r.x, r.y) : (d = a.scrollBy) == null || d.call(a, o, s), {
-        scrolled: this.describeElement(a),
-        scroll: { x: a.scrollLeft, y: a.scrollTop }
+      return o ? (a = f.scrollTo) == null || a.call(f, o.x, o.y) : (d = f.scrollBy) == null || d.call(f, r, c), {
+        scrolled: this.describeElement(f),
+        scroll: { x: f.scrollLeft, y: f.scrollTop }
       };
     }
-    return r ? (w = c.scrollTo) == null || w.call(c, r.x, r.y) : (T = c.scrollBy) == null || T.call(c, o, s), { scroll: { x: c.scrollX, y: c.scrollY } };
+    return o ? (l = i.scrollTo) == null || l.call(i, o.x, o.y) : (u = i.scrollBy) == null || u.call(i, r, c), { scroll: { x: i.scrollX, y: i.scrollY } };
   }
   navigate(e) {
     if (this.options.allowNavigation === !1)
       throw new Error("Navigation is disabled for host page automation.");
-    const n = u(e), r = this.getRoot(), o = p(r), s = g(n.url, "url"), i = new URL(s, o.location.href);
-    if (i.protocol !== "http:" && i.protocol !== "https:")
+    const n = h(e), o = this.getRoot(), r = N(o), c = R(n.url, "url"), s = new URL(c, r.location.href);
+    if (s.protocol !== "http:" && s.protocol !== "https:")
       throw new Error("Navigation only supports HTTP(S) URLs.");
-    return this.refs.clear(), o.location.assign(i.toString()), { navigated: i.toString() };
+    return this.refs.clear(), r.location.assign(s.toString()), { navigated: s.toString() };
+  }
+  hover(e) {
+    const n = h(e), o = this.resolveElement(n), r = L(o).center;
+    return o.dispatchEvent(
+      new MouseEvent("mouseover", {
+        bubbles: !0,
+        cancelable: !0,
+        clientX: r == null ? void 0 : r.x,
+        clientY: r == null ? void 0 : r.y
+      })
+    ), o.dispatchEvent(
+      new MouseEvent("mousemove", {
+        bubbles: !0,
+        cancelable: !0,
+        clientX: r == null ? void 0 : r.x,
+        clientY: r == null ? void 0 : r.y
+      })
+    ), { hovered: this.describeElement(o), point: r };
+  }
+  focus(e) {
+    const n = h(e), o = this.resolveElement(n);
+    return A(o), { focused: this.describeElement(o) };
+  }
+  pointer(e) {
+    const n = h(e), o = F(n.action, ["move", "down", "up", "click"]) ?? "click", r = tt(n.button) ?? 0, c = n.ref || n.selector || n.role || n.name || n.text || n.testId ? this.resolveElement(n) : null, s = c ? L(c).center : {
+      x: M(n.x, "x"),
+      y: M(n.y, "y")
+    };
+    if (!s)
+      throw new Error("Target element has no clickable point.");
+    const i = c ?? g(this.getRoot()).elementFromPoint(s.x, s.y);
+    if (!i)
+      throw new Error(`No element found at (${s.x}, ${s.y}).`);
+    const a = {
+      bubbles: !0,
+      cancelable: !0,
+      clientX: s.x,
+      clientY: s.y,
+      button: r
+    };
+    return o === "move" ? (i.dispatchEvent(_("pointermove", a)), i.dispatchEvent(new MouseEvent("mousemove", a))) : o === "down" ? (i.dispatchEvent(_("pointerdown", a)), i.dispatchEvent(new MouseEvent("mousedown", a))) : o === "up" ? (i.dispatchEvent(_("pointerup", a)), i.dispatchEvent(new MouseEvent("mouseup", a))) : (i.dispatchEvent(_("pointerdown", a)), i.dispatchEvent(new MouseEvent("mousedown", a)), i.dispatchEvent(_("pointerup", a)), i.dispatchEvent(new MouseEvent("mouseup", a)), i instanceof HTMLElement ? i.click() : i.dispatchEvent(new MouseEvent("click", a))), {
+      pointer: o,
+      point: s,
+      target: this.describeElement(i)
+    };
+  }
+  screenshot() {
+    throw new Error(
+      "host_page_screenshot requires the browser extension CDP adapter."
+    );
+  }
+  async waitFor(e) {
+    const n = h(e), o = F(n.state, [
+      "attached",
+      "visible",
+      "hidden",
+      "detached"
+    ]) ?? "visible", r = T(n.timeoutSeconds), c = Math.min(
+      Q,
+      Math.max(0, (r ?? J / 1e3) * 1e3)
+    ), s = Date.now();
+    for (; Date.now() - s <= c; ) {
+      const i = this.tryResolveElement(n);
+      if (o === "attached" ? !!i : o === "detached" ? !i : o === "hidden" ? !i || !m(i) : !!(i && m(i)))
+        return {
+          waitedFor: o,
+          elapsedMs: Date.now() - s,
+          target: i ? this.describeElement(i) : void 0
+        };
+      await new Promise((d) => setTimeout(d, 100));
+    }
+    throw new Error(`Timed out waiting for target to become ${o}.`);
+  }
+  tryResolveElement(e) {
+    try {
+      return this.resolveElement(e);
+    } catch {
+      return null;
+    }
   }
   describeElement(e) {
     return {
       tag: e.tagName.toLowerCase(),
-      role: A(e),
-      name: L(e),
-      selector: M(e)
+      role: x(e),
+      name: E(e),
+      selector: P(e)
     };
   }
 }
-function B(t) {
+function Et(t) {
   return typeof t == "object" && t !== null && !Array.isArray(t) ? t : {};
 }
-function F(t) {
+function mt(t) {
   return JSON.stringify(t);
 }
-function m(t, e, n) {
+function O(t, e, n) {
   return {
     tool_call_id: t.tool_call_id ?? t.id,
     name: t.name,
     status: e,
-    content: F(n)
+    content: mt(n)
   };
 }
-function X(t) {
+function bt(t) {
   return t instanceof Error ? t.message : String(t);
 }
-function G(t = {}) {
-  const e = new j(t);
+function wt(t = {}) {
+  const e = new pt(t);
   return async (n) => {
-    if (!N(n.name))
-      return m(n, "error", {
+    if (!Z(n.name))
+      return O(n, "error", {
         ok: !1,
         error: `Unknown host page automation tool: ${n.name}`
       });
     try {
-      const r = await e.execute(
+      const o = await e.execute(
         n.name,
-        B(n.params)
+        Et(n.params)
       );
-      return m(n, "success", { ok: !0, result: r });
-    } catch (r) {
-      return m(n, "error", {
+      return O(n, "success", { ok: !0, result: o });
+    } catch (o) {
+      return O(n, "error", {
         ok: !1,
-        error: X(r)
+        error: bt(o)
       });
     }
   };
 }
 export {
-  S as HOST_PAGE_AUTOMATION_TOOL_NAMES,
-  j as HostPageAutomationExecutor,
-  G as createHostPageAutomationClientToolHandler,
-  N as isHostPageAutomationToolName
+  z as HOST_PAGE_AUTOMATION_TOOL_NAMES,
+  pt as HostPageAutomationExecutor,
+  wt as createHostPageAutomationClientToolHandler,
+  Z as isHostPageAutomationToolName
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xpert-ai/chatkit-host-automation",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Host page automation tools for Xpert ChatKit client tools",
   "sideEffects": false,
   "type": "module",
@@ -21,7 +21,7 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@xpert-ai/chatkit-types": "~0.3.2"
+    "@xpert-ai/chatkit-types": "~0.3.3"
   },
   "devDependencies": {
     "rimraf": "^6.0.0",