@browserbasehq/stagehand 1.4.0 → 1.5.0

package/lib/dom/debug.ts

@@ -0,0 +1,147 @@
+export async function debugDom() {
+  window.chunkNumber = 0;
+
+  const { selectorMap: multiSelectorMap, outputString } =
+    await window.processElements(window.chunkNumber);
+
+  const selectorMap = multiSelectorMapToSelectorMap(multiSelectorMap);
+
+  drawChunk(selectorMap);
+  setupChunkNav();
+}
+
+function multiSelectorMapToSelectorMap(
+  multiSelectorMap: Record<number, string[]>,
+) {
+  return Object.fromEntries(
+    Object.entries(multiSelectorMap).map(([key, selectors]) => [
+      Number(key),
+      selectors[0],
+    ]),
+  );
+}
+
+function drawChunk(selectorMap: Record<number, string>) {
+  if (!window.showChunks) return;
+  cleanupMarkers();
+  Object.entries(selectorMap).forEach(([_index, selector]) => {
+    const element = document.evaluate(
+      selector as string,
+      document,
+      null,
+      XPathResult.FIRST_ORDERED_NODE_TYPE,
+      null,
+    ).singleNodeValue as Element;
+
+    if (element) {
+      let rect;
+      if (element.nodeType === Node.ELEMENT_NODE) {
+        rect = element.getBoundingClientRect();
+      } else {
+        const range = document.createRange();
+        range.selectNodeContents(element);
+        rect = range.getBoundingClientRect();
+      }
+      const color = "grey";
+      const overlay = document.createElement("div");
+      overlay.style.position = "absolute";
+      overlay.style.left = `${rect.left + window.scrollX}px`;
+      overlay.style.top = `${rect.top + window.scrollY}px`;
+      overlay.style.padding = "2px"; // Add 2px of padding to the overlay
+
+      overlay.style.width = `${rect.width}px`;
+      overlay.style.height = `${rect.height}px`;
+      overlay.style.backgroundColor = color;
+      overlay.className = "stagehand-marker";
+      overlay.style.opacity = "0.3";
+      overlay.style.zIndex = "1000000000"; // Ensure it's above the element
+      overlay.style.border = "1px solid"; // Add a 1px solid border to the overlay
+      overlay.style.pointerEvents = "none"; // Ensure the overlay does not capture mouse events
+      document.body.appendChild(overlay);
+    }
+  });
+}
+
+async function cleanupDebug() {
+  cleanupMarkers();
+  cleanupNav();
+}
+
+function cleanupMarkers() {
+  const markers = document.querySelectorAll(".stagehand-marker");
+  markers.forEach((marker) => {
+    marker.remove();
+  });
+}
+
+function cleanupNav() {
+  const stagehandNavElements = document.querySelectorAll(".stagehand-nav");
+  stagehandNavElements.forEach((element) => {
+    element.remove();
+  });
+}
+
+function setupChunkNav() {
+  const viewportHeight = window.innerHeight;
+  const documentHeight = document.documentElement.scrollHeight;
+  const totalChunks = Math.ceil(documentHeight / viewportHeight);
+
+  if (window.chunkNumber > 0) {
+    const prevChunkButton = document.createElement("button");
+    prevChunkButton.className = "stagehand-nav";
+
+    prevChunkButton.textContent = "Previous";
+    prevChunkButton.style.marginLeft = "50px";
+    prevChunkButton.style.position = "fixed";
+    prevChunkButton.style.bottom = "10px";
+    prevChunkButton.style.left = "50%";
+    prevChunkButton.style.transform = "translateX(-50%)";
+    prevChunkButton.style.zIndex = "1000000000";
+    prevChunkButton.onclick = async () => {
+      cleanupMarkers();
+      cleanupNav();
+      window.chunkNumber -= 1;
+      window.scrollTo(0, window.chunkNumber * window.innerHeight);
+      await window.waitForDomSettle();
+      const { selectorMap: multiSelectorMap } = await window.processElements(
+        window.chunkNumber,
+      );
+
+      const selectorMap = multiSelectorMapToSelectorMap(multiSelectorMap);
+
+      drawChunk(selectorMap);
+      setupChunkNav();
+    };
+    document.body.appendChild(prevChunkButton);
+  }
+  if (totalChunks > window.chunkNumber) {
+    const nextChunkButton = document.createElement("button");
+    nextChunkButton.className = "stagehand-nav";
+    nextChunkButton.textContent = "Next";
+    nextChunkButton.style.marginRight = "50px";
+    nextChunkButton.style.position = "fixed";
+    nextChunkButton.style.bottom = "10px";
+    nextChunkButton.style.right = "50%";
+    nextChunkButton.style.transform = "translateX(50%)";
+    nextChunkButton.style.zIndex = "1000000000";
+    nextChunkButton.onclick = async () => {
+      cleanupMarkers();
+      cleanupNav();
+      window.chunkNumber += 1;
+      window.scrollTo(0, window.chunkNumber * window.innerHeight);
+      await window.waitForDomSettle();
+
+      const { selectorMap: multiSelectorMap } = await window.processElements(
+        window.chunkNumber,
+      );
+      const selectorMap = multiSelectorMapToSelectorMap(multiSelectorMap);
+      drawChunk(selectorMap);
+      setupChunkNav();
+    };
+
+    document.body.appendChild(nextChunkButton);
+  }
+}
+
+window.debugDom = debugDom;
+window.cleanupDebug = cleanupDebug;

package/lib/dom/genDomScripts.ts

@@ -0,0 +1,29 @@
+/**
+ * We have a collection of typescript functions that we need to run in the browser.
+ * First, we build them into a single js file
+ * Second, due to framework differences we need to get our script content as a string to avoid pathing issues due to file routing in frameworks like Next.js
+ * Playwright allows us to pass in script content directly as a string instead of reading a file from a path
+ * https://github.com/browserbase/stagehand/issues/180
+ *
+ * We can't rely on the normal build process for stagehand, because we need our script content as a string so that the import *just works*
+ */
+import fs from "fs";
+import path from "path";
+import esbuild from "esbuild";
+
+fs.mkdirSync(path.join(__dirname, "./build"), { recursive: true });
+
+esbuild.buildSync({
+  entryPoints: [path.join(__dirname, "index.ts")],
+  bundle: true,
+  outdir: path.join(__dirname, "build"),
+});
+
+const scriptContent = fs.readFileSync(
+  path.join(__dirname, "./build/index.js"),
+  "utf8",
+);
+
+const output = `export const scriptContent = ${JSON.stringify(scriptContent)};`;
+
+fs.writeFileSync(path.join(__dirname, "./build/scriptContent.ts"), output);

package/lib/dom/global.d.ts

@@ -0,0 +1,25 @@
+export {};
+declare global {
+  interface Window {
+    chunkNumber: number;
+    showChunks?: boolean;
+    processDom: (chunksSeen: Array<number>) => Promise<{
+      outputString: string;
+      selectorMap: Record<number, string[]>;
+      chunk: number;
+      chunks: number[];
+    }>;
+    processAllOfDom: () => Promise<{
+      outputString: string;
+      selectorMap: Record<number, string[]>;
+    }>;
+    processElements: (chunk: number) => Promise<{
+      outputString: string;
+      selectorMap: Record<number, string[]>;
+    }>;
+    debugDom: () => Promise<void>;
+    cleanupDebug: () => void;
+    scrollToHeight: (height: number) => Promise<void>;
+    waitForDomSettle: () => Promise<void>;
+  }
+}

package/lib/dom/index.ts

@@ -0,0 +1,3 @@
+export * from "./process";
+export * from "./utils";
+export * from "./debug";

package/lib/dom/process.ts

@@ -0,0 +1,441 @@
+import { generateXPathsForElement as generateXPaths } from "./xpathUtils";
+
+export function isElementNode(node: Node): node is Element {
+  return node.nodeType === Node.ELEMENT_NODE;
+}
+
+export function isTextNode(node: Node): node is Text {
+  return node.nodeType === Node.TEXT_NODE && Boolean(node.textContent?.trim());
+}
+
+export async function processDom(chunksSeen: Array<number>) {
+  const { chunk, chunksArray } = await pickChunk(chunksSeen);
+  const { outputString, selectorMap } = await processElements(chunk);
+
+  console.log(
+    `Stagehand (Browser Process): Extracted dom elements:\n${outputString}`,
+  );
+
+  return {
+    outputString,
+    selectorMap,
+    chunk,
+    chunks: chunksArray,
+  };
+}
+
+export async function processAllOfDom() {
+  console.log("Stagehand (Browser Process): Processing all of DOM");
+
+  const viewportHeight = window.innerHeight;
+  const documentHeight = document.documentElement.scrollHeight;
+  const totalChunks = Math.ceil(documentHeight / viewportHeight);
+
+  let index = 0;
+  const results = [];
+  for (let chunk = 0; chunk < totalChunks; chunk++) {
+    const result = await processElements(chunk, true, index);
+    results.push(result);
+    index += Object.keys(result.selectorMap).length;
+  }
+
+  await scrollToHeight(0);
+
+  const allOutputString = results.map((result) => result.outputString).join("");
+  const allSelectorMap = results.reduce(
+    (acc, result) => ({ ...acc, ...result.selectorMap }),
+    {},
+  );
+
+  console.log(
+    `Stagehand (Browser Process): All dom elements: ${allOutputString}`,
+  );
+
+  return {
+    outputString: allOutputString,
+    selectorMap: allSelectorMap,
+  };
+}
+
+export async function scrollToHeight(height: number) {
+  window.scrollTo({ top: height, left: 0, behavior: "smooth" });
+
+  // Wait for scrolling to finish using the scrollend event
+  await new Promise<void>((resolve) => {
+    let scrollEndTimer: number;
+    const handleScrollEnd = () => {
+      clearTimeout(scrollEndTimer);
+      scrollEndTimer = window.setTimeout(() => {
+        window.removeEventListener("scroll", handleScrollEnd);
+        resolve();
+      }, 100);
+    };
+
+    window.addEventListener("scroll", handleScrollEnd, { passive: true });
+    handleScrollEnd();
+  });
+}
+
+const xpathCache: Map<Node, string[]> = new Map();
+
+export async function processElements(
+  chunk: number,
+  scrollToChunk: boolean = true,
+  indexOffset: number = 0,
+): Promise<{
+  outputString: string;
+  selectorMap: Record<number, string[]>;
+}> {
+  console.time("processElements:total");
+  const viewportHeight = window.innerHeight;
+  const chunkHeight = viewportHeight * chunk;
+
+  // Calculate the maximum scrollable offset
+  const maxScrollTop =
+    document.documentElement.scrollHeight - window.innerHeight;
+
+  // Adjust the offsetTop to not exceed the maximum scrollable offset
+  const offsetTop = Math.min(chunkHeight, maxScrollTop);
+
+  if (scrollToChunk) {
+    console.time("processElements:scroll");
+    await scrollToHeight(offsetTop);
+    console.timeEnd("processElements:scroll");
+  }
+
+  const candidateElements: Array<ChildNode> = [];
+  const DOMQueue: Array<ChildNode> = [...document.body.childNodes];
+
+  console.log("Stagehand (Browser Process): Generating candidate elements");
+  console.time("processElements:findCandidates");
+
+  while (DOMQueue.length > 0) {
+    const element = DOMQueue.pop();
+
+    let shouldAddElement = false;
+
+    if (element && isElementNode(element)) {
+      const childrenCount = element.childNodes.length;
+
+      // Always traverse child nodes
+      for (let i = childrenCount - 1; i >= 0; i--) {
+        const child = element.childNodes[i];
+        DOMQueue.push(child as ChildNode);
+      }
+
+      // Check if element is interactive
+      if (isInteractiveElement(element)) {
+        if (isActive(element) && isVisible(element)) {
+          shouldAddElement = true;
+        }
+      }
+
+      if (isLeafElement(element)) {
+        if (isActive(element) && isVisible(element)) {
+          shouldAddElement = true;
+        }
+      }
+    }
+
+    if (element && isTextNode(element) && isTextVisible(element)) {
+      shouldAddElement = true;
+    }
+
+    if (shouldAddElement) {
+      candidateElements.push(element);
+    }
+  }
+
+  console.timeEnd("processElements:findCandidates");
+
+  const selectorMap: Record<number, string[]> = {};
+  let outputString = "";
+
+  console.log(
+    `Stagehand (Browser Process): Processing candidate elements: ${candidateElements.length}`,
+  );
+
+  console.time("processElements:processCandidates");
+  console.time("processElements:generateXPaths");
+  const xpathLists = await Promise.all(
+    candidateElements.map(async (element) => {
+      if (xpathCache.has(element)) {
+        return xpathCache.get(element);
+      }
+
+      const xpaths = await generateXPaths(element);
+      xpathCache.set(element, xpaths);
+      return xpaths;
+    }),
+  );
+  console.timeEnd("processElements:generateXPaths");
+
+  candidateElements.forEach((element, index) => {
+    const xpaths = xpathLists[index];
+    let elementOutput = "";
+
+    if (isTextNode(element)) {
+      const textContent = element.textContent?.trim();
+      if (textContent) {
+        elementOutput += `${index + indexOffset}:${textContent}\n`;
+      }
+    } else if (isElementNode(element)) {
+      const tagName = element.tagName.toLowerCase();
+      const attributes = collectEssentialAttributes(element);
+
+      const openingTag = `<${tagName}${attributes ? " " + attributes : ""}>`;
+      const closingTag = `</${tagName}>`;
+      const textContent = element.textContent?.trim() || "";
+
+      elementOutput += `${index + indexOffset}:${openingTag}${textContent}${closingTag}\n`;
+    }
+
+    outputString += elementOutput;
+    selectorMap[index + indexOffset] = xpaths;
+  });
+  console.timeEnd("processElements:processCandidates");
+
+  console.timeEnd("processElements:total");
+  return {
+    outputString,
+    selectorMap,
+  };
+}
+
+/**
+ * Collects essential attributes from an element.
+ * @param element The DOM element.
+ * @returns A string of formatted attributes.
+ */
+function collectEssentialAttributes(element: Element): string {
+  const essentialAttributes = [
+    "id",
+    "class",
+    "href",
+    "src",
+    "aria-label",
+    "aria-name",
+    "aria-role",
+    "aria-description",
+    "aria-expanded",
+    "aria-haspopup",
+    "type",
+    "value",
+  ];
+
+  const attrs: string[] = essentialAttributes
+    .map((attr) => {
+      const value = element.getAttribute(attr);
+      return value ? `${attr}="${value}"` : "";
+    })
+    .filter((attr) => attr !== "");
+
+  // Collect data- attributes
+  Array.from(element.attributes).forEach((attr) => {
+    if (attr.name.startsWith("data-")) {
+      attrs.push(`${attr.name}="${attr.value}"`);
+    }
+  });
+
+  return attrs.join(" ");
+}
+
+window.processDom = processDom;
+window.processAllOfDom = processAllOfDom;
+window.processElements = processElements;
+window.scrollToHeight = scrollToHeight;
+
+const leafElementDenyList = ["SVG", "IFRAME", "SCRIPT", "STYLE", "LINK"];
+
+const interactiveElementTypes = [
+  "A",
+  "BUTTON",
+  "DETAILS",
+  "EMBED",
+  "INPUT",
+  "LABEL",
+  "MENU",
+  "MENUITEM",
+  "OBJECT",
+  "SELECT",
+  "TEXTAREA",
+  "SUMMARY",
+];
+
+const interactiveRoles = [
+  "button",
+  "menu",
+  "menuitem",
+  "link",
+  "checkbox",
+  "radio",
+  "slider",
+  "tab",
+  "tabpanel",
+  "textbox",
+  "combobox",
+  "grid",
+  "listbox",
+  "option",
+  "progressbar",
+  "scrollbar",
+  "searchbox",
+  "switch",
+  "tree",
+  "treeitem",
+  "spinbutton",
+  "tooltip",
+];
+const interactiveAriaRoles = ["menu", "menuitem", "button"];
+
+/*
+ * Checks if an element is visible and therefore relevant for LLMs to consider. We check:
+ * - Size
+ * - Display properties
+ * - Opacity
+ * If the element is a child of a previously hidden element, it should not be included, so we don't consider downstream effects of a parent element here
+ */
+const isVisible = (element: Element) => {
+  const rect = element.getBoundingClientRect();
+  // Ensure the element is within the viewport
+  if (
+    rect.width === 0 ||
+    rect.height === 0 ||
+    rect.top < 0 ||
+    rect.top > window.innerHeight
+  ) {
+    return false;
+  }
+  if (!isTopElement(element, rect)) {
+    return false;
+  }
+
+  const visible = element.checkVisibility({
+    checkOpacity: true,
+    checkVisibilityCSS: true,
+  });
+
+  return visible;
+};
+
+const isTextVisible = (element: ChildNode) => {
+  const range = document.createRange();
+  range.selectNodeContents(element);
+  const rect = range.getBoundingClientRect();
+
+  if (
+    rect.width === 0 ||
+    rect.height === 0 ||
+    rect.top < 0 ||
+    rect.top > window.innerHeight
+  ) {
+    return false;
+  }
+  const parent = element.parentElement;
+  if (!parent) {
+    return false;
+  }
+  if (!isTopElement(parent, rect)) {
+    return false;
+  }
+
+  const visible = parent.checkVisibility({
+    checkOpacity: true,
+    checkVisibilityCSS: true,
+  });
+
+  return visible;
+};
+
+function isTopElement(elem: ChildNode, rect: DOMRect) {
+  const points = [
+    { x: rect.left + rect.width * 0.25, y: rect.top + rect.height * 0.25 },
+    { x: rect.left + rect.width * 0.75, y: rect.top + rect.height * 0.25 },
+    { x: rect.left + rect.width * 0.25, y: rect.top + rect.height * 0.75 },
+    { x: rect.left + rect.width * 0.75, y: rect.top + rect.height * 0.75 },
+    { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 },
+  ];
+
+  return points.some((point) => {
+    const topEl = document.elementFromPoint(point.x, point.y);
+    let current = topEl;
+    while (current && current !== document.body) {
+      if (current.isSameNode(elem)) {
+        return true;
+      }
+      current = current.parentElement;
+    }
+    return false;
+  });
+}
+
+const isActive = (element: Element) => {
+  if (
+    element.hasAttribute("disabled") ||
+    element.hasAttribute("hidden") ||
+    element.getAttribute("aria-disabled") === "true"
+  ) {
+    return false;
+  }
+
+  return true;
+};
+const isInteractiveElement = (element: Element) => {
+  const elementType = element.tagName;
+  const elementRole = element.getAttribute("role");
+  const elementAriaRole = element.getAttribute("aria-role");
+
+  return (
+    (elementType && interactiveElementTypes.includes(elementType)) ||
+    (elementRole && interactiveRoles.includes(elementRole)) ||
+    (elementAriaRole && interactiveAriaRoles.includes(elementAriaRole))
+  );
+};
+
+const isLeafElement = (element: Element) => {
+  if (element.textContent === "") {
+    return false;
+  }
+
+  if (element.childNodes.length === 0) {
+    return !leafElementDenyList.includes(element.tagName);
+  }
+
+  // This case ensures that extra context will be included for simple element nodes that contain only text
+  if (element.childNodes.length === 1 && isTextNode(element.childNodes[0])) {
+    return true;
+  }
+
+  return false;
+};
+
+async function pickChunk(chunksSeen: Array<number>) {
+  const viewportHeight = window.innerHeight;
+  const documentHeight = document.documentElement.scrollHeight;
+
+  const chunks = Math.ceil(documentHeight / viewportHeight);
+
+  const chunksArray = Array.from({ length: chunks }, (_, i) => i);
+  const chunksRemaining = chunksArray.filter((chunk) => {
+    return !chunksSeen.includes(chunk);
+  });
+
+  const currentScrollPosition = window.scrollY;
+  const closestChunk = chunksRemaining.reduce((closest, current) => {
+    const currentChunkTop = viewportHeight * current;
+    const closestChunkTop = viewportHeight * closest;
+    return Math.abs(currentScrollPosition - currentChunkTop) <
+      Math.abs(currentScrollPosition - closestChunkTop)
+      ? current
+      : closest;
+  }, chunksRemaining[0]);
+  const chunk = closestChunk;
+
+  if (chunk === undefined) {
+    throw new Error(`No chunks remaining to check: ${chunksRemaining}`);
+  }
+  return {
+    chunk,
+    chunksArray,
+  };
+}

package/lib/dom/utils.ts

@@ -0,0 +1,17 @@
+export async function waitForDomSettle() {
+  return new Promise<void>((resolve) => {
+    const createTimeout = () => {
+      return setTimeout(() => {
+        resolve();
+      }, 2000);
+    };
+    let timeout = createTimeout();
+    const observer = new MutationObserver(() => {
+      clearTimeout(timeout);
+      timeout = createTimeout();
+    });
+    observer.observe(window.document.body, { childList: true, subtree: true });
+  });
+}
+
+window.waitForDomSettle = waitForDomSettle;