agent-browser-loop 0.2.0 → 0.2.2

package/README.md

@@ -172,6 +172,8 @@ export default defineBrowserConfig({
 });
 ```
 
+On macOS, headless system Chrome can crash during AppKit startup. By default, the CLI falls back to bundled Playwright Chromium when `headless: true`. If you explicitly want system Chrome in headless mode, set `allowSystemChromeHeadless: true`.
+
 ## What This Is NOT For
 
 This tool is for agents to test their own code. It is **not** for:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-browser-loop",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "Let your AI coding agent drive a browser to verify its own work",
   "license": "MIT",
   "author": "Jason Silberman",

package/src/actions.ts

@@ -1,4 +1,5 @@
 import type { Page, Request } from "playwright";
+import type { ElementRefStore } from "./ref-store";
 import type {
   ClickOptions,
   NavigateOptions,
@@ -7,27 +8,26 @@ import type {
 } from "./types";
 
 /**
- * Get a locator for an element by ref or index
- * After calling getState(), elements have data-ref attributes injected
+ * Get a locator for an element by ref or index using the ref store
+ * The ref store contains selectors generated during getState()
  */
-function getLocator(page: Page, options: { ref?: string; index?: number }) {
-  if (options.ref) {
-    return page.locator(`[data-ref="${options.ref}"]`);
-  }
-  if (options.index !== undefined) {
-    // Use data-index (injected by getState). Fallback to legacy e{index} refs.
-    return page.locator(
-      `[data-index="${options.index}"], [data-ref="e${options.index}"]`,
-    );
-  }
-  throw new Error("Must provide either ref or index");
+async function getLocator(
+  page: Page,
+  refStore: ElementRefStore,
+  options: { ref?: string; index?: number },
+) {
+  return await refStore.resolveLocator(page, options);
 }
 
 /**
  * Click an element
  */
-export async function click(page: Page, options: ClickOptions): Promise<void> {
-  const locator = getLocator(page, options);
+export async function click(
+  page: Page,
+  refStore: ElementRefStore,
+  options: ClickOptions,
+): Promise<void> {
+  const locator = await getLocator(page, refStore, options);
 
   const clickOptions: Parameters<typeof locator.click>[0] = {
     button: options.button,
@@ -44,8 +44,12 @@ export async function click(page: Page, options: ClickOptions): Promise<void> {
 /**
  * Type text into an element
  */
-export async function type(page: Page, options: TypeOptions): Promise<void> {
-  const locator = getLocator(page, options);
+export async function type(
+  page: Page,
+  refStore: ElementRefStore,
+  options: TypeOptions,
+): Promise<void> {
+  const locator = await getLocator(page, refStore, options);
 
   // Clear existing text if requested
   if (options.clear) {
@@ -129,9 +133,10 @@ export async function waitForElement(
  */
 export async function hover(
   page: Page,
+  refStore: ElementRefStore,
   options: { ref?: string; index?: number },
 ): Promise<void> {
-  const locator = getLocator(page, options);
+  const locator = await getLocator(page, refStore, options);
   await locator.hover();
 }
 
@@ -140,9 +145,10 @@ export async function hover(
  */
 export async function select(
   page: Page,
+  refStore: ElementRefStore,
   options: { ref?: string; index?: number; value: string | string[] },
 ): Promise<void> {
-  const locator = getLocator(page, options);
+  const locator = await getLocator(page, refStore, options);
   await locator.selectOption(options.value);
 }
 

package/src/browser.ts

@@ -3,6 +3,7 @@ import { chromium } from "playwright";
 import * as actions from "./actions";
 import { findChromeExecutable } from "./chrome";
 import { log } from "./log";
+import { ElementRefStore } from "./ref-store";
 import { formatStateText, getState } from "./state";
 import type {
   BrowserConfig,
@@ -33,12 +34,14 @@ export class AgentBrowser {
   private networkLogLimit: number;
   private usePersistentContext = false;
   private lastState: BrowserState | null = null;
+  private refStore: ElementRefStore = new ElementRefStore();
 
   constructor(options: AgentBrowserOptions = {}) {
     this.config = {
       headless: options.headless ?? true,
       executablePath: options.executablePath,
       useSystemChrome: options.useSystemChrome ?? true,
+      allowSystemChromeHeadless: options.allowSystemChromeHeadless,
       viewportWidth: options.viewportWidth ?? 1280,
       viewportHeight: options.viewportHeight ?? 720,
       userDataDir: options.userDataDir,
@@ -60,8 +63,27 @@ export class AgentBrowser {
       throw new Error("Browser already started");
     }
 
-    const resolvedExecutablePath = this.config.useSystemChrome
-      ? this.config.executablePath || findChromeExecutable()
+    const isDarwin = process.platform === "darwin";
+    let useSystemChrome = this.config.useSystemChrome ?? true;
+    let executablePath = this.config.executablePath;
+
+    if (
+      isDarwin &&
+      this.config.headless &&
+      (useSystemChrome || executablePath) &&
+      !this.config.allowSystemChromeHeadless
+    ) {
+      log
+        .withMetadata({ executablePath })
+        .warn(
+          "Headless system Chrome can crash on macOS. Falling back to bundled Chromium. Set allowSystemChromeHeadless to true to override.",
+        );
+      useSystemChrome = false;
+      executablePath = undefined;
+    }
+
+    const resolvedExecutablePath = useSystemChrome
+      ? executablePath || findChromeExecutable()
       : undefined;
 
     log
@@ -160,6 +182,7 @@ export class AgentBrowser {
     this.networkLogs = [];
     this.networkCaptureEnabled = false;
     this.usePersistentContext = false;
+    this.refStore.clear();
   }
 
   /**
@@ -190,15 +213,21 @@ export class AgentBrowser {
     options?: Omit<NavigateOptions, "url">,
   ): Promise<void> {
     await actions.navigate(this.getPage(), { url, ...options });
+    this.refStore.clear();
   }
 
   /**
    * Get rich state of the current page
-   * Also injects data-ref attributes for element targeting
+   * Stores element refs server-side (no DOM modification)
    */
   async getState(options?: GetStateOptions): Promise<BrowserState> {
-    // getState now handles ref injection internally
-    const state = await getState(this.getPage(), this.getContext(), options);
+    // getState now stores refs in this.refStore instead of injecting into DOM
+    const state = await getState(
+      this.getPage(),
+      this.getContext(),
+      this.refStore,
+      options,
+    );
     const result = {
       ...state,
       errors: {
@@ -254,14 +283,14 @@ export class AgentBrowser {
    * Click an element
    */
   async click(options: ClickOptions): Promise<void> {
-    await actions.click(this.getPage(), options);
+    await actions.click(this.getPage(), this.refStore, options);
   }
 
   /**
    * Type text into an element
    */
   async type(options: TypeOptions): Promise<void> {
-    await actions.type(this.getPage(), options);
+    await actions.type(this.getPage(), this.refStore, options);
   }
 
   /**
@@ -429,7 +458,7 @@ export class AgentBrowser {
    * Hover over an element
    */
   async hover(options: { ref?: string; index?: number }): Promise<void> {
-    await actions.hover(this.getPage(), options);
+    await actions.hover(this.getPage(), this.refStore, options);
   }
 
   /**
@@ -440,7 +469,7 @@ export class AgentBrowser {
     index?: number;
     value: string | string[];
   }): Promise<void> {
-    await actions.select(this.getPage(), options);
+    await actions.select(this.getPage(), this.refStore, options);
   }
 
   /**
@@ -554,6 +583,13 @@ export class AgentBrowser {
     }
     return state;
   }
+
+  /**
+   * Get the element ref store (for advanced usage/testing)
+   */
+  getRefStore(): ElementRefStore {
+    return this.refStore;
+  }
 }
 
 /**

package/src/cli.ts

@@ -147,6 +147,7 @@ async function resolveBrowserOptions(args: {
     headless,
     executablePath: config?.executablePath,
     useSystemChrome,
+    allowSystemChromeHeadless: config?.allowSystemChromeHeadless,
     viewportWidth: config?.viewportWidth,
     viewportHeight: config?.viewportHeight,
     userDataDir: config?.userDataDir,
@@ -321,7 +322,10 @@ const openCommand = command({
     json: jsonFlag,
   },
   handler: async (args) => {
-    const browserOptions = await resolveBrowserOptions(args);
+    const browserOptions = await resolveBrowserOptions({
+      ...args,
+      configPath: args.config,
+    });
 
     let client: DaemonClient;
     if (args.new) {
@@ -386,7 +390,10 @@ const actCommand = command({
       process.exit(1);
     }
 
-    const browserOptions = await resolveBrowserOptions(args);
+    const browserOptions = await resolveBrowserOptions({
+      ...args,
+      configPath: args.config,
+    });
 
     let client: DaemonClient;
     if (args.new) {
@@ -581,16 +588,20 @@ const screenshotCommand = command({
       process.exit(1);
     }
 
-    const data = response.data as { base64: string };
+    // Handle both raw string (from executeCommand) and object format
+    const base64 =
+      typeof response.data === "string"
+        ? response.data
+        : (response.data as { base64: string }).base64;
 
     if (args.output) {
       // Write to file
-      const buffer = Buffer.from(data.base64, "base64");
+      const buffer = Buffer.from(base64, "base64");
       await Bun.write(args.output, buffer);
       console.log(`Screenshot saved to ${args.output}`);
     } else {
       // Output base64
-      console.log(data.base64);
+      console.log(base64);
     }
   },
 });
@@ -829,6 +840,7 @@ const serverCommand = command({
         headless,
         executablePath: args.executablePath ?? config?.executablePath,
         useSystemChrome,
+        allowSystemChromeHeadless: config?.allowSystemChromeHeadless,
         viewportWidth: args.viewportWidth || config?.viewportWidth,
         viewportHeight: args.viewportHeight || config?.viewportHeight,
         userDataDir: args.userDataDir ?? config?.userDataDir,

package/src/config.ts

@@ -32,6 +32,7 @@ export const browserCliConfigSchema = z.looseObject({
   headless: z.boolean().optional(),
   executablePath: z.string().optional(),
   useSystemChrome: z.boolean().optional(),
+  allowSystemChromeHeadless: z.boolean().optional(),
   viewportWidth: z.number().int().optional(),
   viewportHeight: z.number().int().optional(),
   userDataDir: z.string().optional(),

package/src/daemon.ts

@@ -239,14 +239,6 @@ export async function startDaemon(options: DaemonOptions = {}): Promise<void> {
     return session;
   }
 
-  function getSession(sessionId: string): DaemonSession {
-    const session = sessions.get(sessionId);
-    if (!session) {
-      throw new Error(`Session not found: ${sessionId}`);
-    }
-    return session;
-  }
-
   function getOrDefaultSession(sessionId?: string): DaemonSession {
     const id = sessionId ?? "default";
     const session = sessions.get(id);
@@ -333,16 +325,13 @@ export async function startDaemon(options: DaemonOptions = {}): Promise<void> {
               parseResult.data,
               sessions,
               createSession,
-              getSession,
               getOrDefaultSession,
               closeSession,
-              idGenerator,
-              defaultOptions,
             );
 
             // Handle shutdown
             if (parseResult.data.type === "shutdown") {
-              socket.write(JSON.stringify(response) + "\n");
+              socket.write(`${JSON.stringify(response)}\n`);
               shutdown();
               return;
             }
@@ -355,7 +344,7 @@ export async function startDaemon(options: DaemonOptions = {}): Promise<void> {
           };
         }
 
-        socket.write(JSON.stringify(response) + "\n");
+        socket.write(`${JSON.stringify(response)}\n`);
       }
     });
 
@@ -411,11 +400,8 @@ async function handleRequest(
     sessionId?: string,
     options?: AgentBrowserOptions,
   ) => Promise<DaemonSession>,
-  getSession: (sessionId: string) => DaemonSession,
   getOrDefaultSession: (sessionId?: string) => DaemonSession,
   closeSession: (sessionId: string) => Promise<void>,
-  idGenerator: ReturnType<typeof createIdGenerator>,
-  defaultOptions: AgentBrowserOptions,
 ): Promise<DaemonResponse> {
   const { id } = request;
 
@@ -613,7 +599,7 @@ export class DaemonClient {
       let buffer = "";
 
       socket.on("connect", () => {
-        socket.write(JSON.stringify(request) + "\n");
+        socket.write(`${JSON.stringify(request)}\n`);
       });
 
       socket.on("data", (data) => {
@@ -914,7 +900,7 @@ async function spawnDaemon(
 
   const child = spawn(
     process.execPath,
-    ["--bun", import.meta.dirname + "/daemon-entry.ts", "--config", configPath],
+    ["--bun", `${import.meta.dirname}/daemon-entry.ts`, "--config", configPath],
     {
       detached: true,
       stdio: "ignore",

package/src/index.ts

@@ -35,6 +35,9 @@ export {
   isDaemonRunning,
   startDaemon,
 } from "./daemon";
+export type { ElementSelectors, StoredElementRef } from "./ref-store";
+// Ref store for server-side element reference management
+export { ElementRefStore } from "./ref-store";
 // Server
 export type { BrowserServerConfig } from "./server";
 export { startBrowserServer } from "./server";

package/src/ref-store.ts

@@ -0,0 +1,216 @@
+import type { Locator, Page } from "playwright";
+
+/**
+ * Selector strategies for locating an element
+ * Multiple strategies provide resilience if one fails
+ */
+export interface ElementSelectors {
+  /** XPath from document root */
+  xpath: string;
+  /** CSS selector path */
+  cssPath: string;
+  /** Fingerprint-based selector using stable attributes */
+  fingerprint?: string;
+}
+
+/**
+ * Stored reference to an element
+ */
+export interface StoredElementRef {
+  /** The ref string (e.g., "button_0") */
+  ref: string;
+  /** Sequential index */
+  index: number;
+  /** Multiple selector strategies */
+  selectors: ElementSelectors;
+  /** Element fingerprint for validation */
+  fingerprint: {
+    tagName: string;
+    role?: string;
+    type?: string;
+    name?: string;
+    placeholder?: string;
+  };
+}
+
+/**
+ * Server-side store for element references
+ * Avoids DOM modification that causes React hydration errors
+ */
+export class ElementRefStore {
+  private refMap = new Map<string, StoredElementRef>();
+  private indexMap = new Map<number, StoredElementRef>();
+  private snapshotVersion = 0;
+
+  /**
+   * Clear all stored refs (call before new snapshot)
+   */
+  clear(): void {
+    this.refMap.clear();
+    this.indexMap.clear();
+    this.snapshotVersion++;
+  }
+
+  /**
+   * Get current snapshot version
+   */
+  getVersion(): number {
+    return this.snapshotVersion;
+  }
+
+  /**
+   * Store a ref for an element
+   */
+  set(
+    ref: string,
+    index: number,
+    selectors: ElementSelectors,
+    fingerprint: StoredElementRef["fingerprint"],
+  ): void {
+    const stored: StoredElementRef = { ref, index, selectors, fingerprint };
+    this.refMap.set(ref, stored);
+    this.indexMap.set(index, stored);
+  }
+
+  /**
+   * Get stored ref by ref string
+   */
+  getByRef(ref: string): StoredElementRef | undefined {
+    return this.refMap.get(ref);
+  }
+
+  /**
+   * Get stored ref by index
+   */
+  getByIndex(index: number): StoredElementRef | undefined {
+    return this.indexMap.get(index);
+  }
+
+  /**
+   * Resolve a Playwright locator for an element by ref or index
+   */
+  async resolveLocator(
+    page: Page,
+    options: { ref?: string; index?: number },
+  ): Promise<Locator> {
+    let stored: StoredElementRef | undefined;
+
+    if (options.ref) {
+      stored = this.refMap.get(options.ref);
+      if (!stored) {
+        throw new Error(
+          `Unknown ref: ${options.ref}. Call getState() first to snapshot elements.`,
+        );
+      }
+    } else if (options.index !== undefined) {
+      stored = this.indexMap.get(options.index);
+      if (!stored) {
+        throw new Error(
+          `Unknown index: ${options.index}. Call getState() first to snapshot elements.`,
+        );
+      }
+    } else {
+      throw new Error("Must provide either ref or index");
+    }
+
+    const pickMatching = async (locator: Locator): Promise<Locator | null> => {
+      const count = await locator.count();
+      if (count === 0) {
+        return null;
+      }
+
+      for (let i = 0; i < count; i++) {
+        const candidate = locator.nth(i);
+        const matches = await candidate.evaluate((el, fingerprint) => {
+          const element = el as HTMLElement;
+          if (
+            fingerprint.tagName &&
+            element.tagName.toLowerCase() !== fingerprint.tagName
+          ) {
+            return false;
+          }
+          if (
+            fingerprint.role &&
+            element.getAttribute("role") !== fingerprint.role
+          ) {
+            return false;
+          }
+          if (
+            fingerprint.type &&
+            element.getAttribute("type") !== fingerprint.type
+          ) {
+            return false;
+          }
+          if (
+            fingerprint.name &&
+            element.getAttribute("name") !== fingerprint.name
+          ) {
+            return false;
+          }
+          if (
+            fingerprint.placeholder &&
+            element.getAttribute("placeholder") !== fingerprint.placeholder
+          ) {
+            return false;
+          }
+          return true;
+        }, stored!.fingerprint);
+
+        if (matches) {
+          return candidate;
+        }
+      }
+
+      return null;
+    };
+
+    const selectors = stored.selectors;
+
+    const xpathLocator = page.locator(`xpath=${selectors.xpath}`);
+    const xpathMatch = await pickMatching(xpathLocator);
+    if (xpathMatch) {
+      return xpathMatch;
+    }
+
+    const cssLocator = page.locator(selectors.cssPath);
+    const cssMatch = await pickMatching(cssLocator);
+    if (cssMatch) {
+      return cssMatch;
+    }
+
+    let fingerprintLocator: Locator | null = null;
+    if (selectors.fingerprint) {
+      const tagPrefix = stored.fingerprint.tagName || "";
+      const fingerprintSelector = selectors.fingerprint.startsWith("[")
+        ? `${tagPrefix}${selectors.fingerprint}`
+        : selectors.fingerprint;
+      fingerprintLocator = page.locator(fingerprintSelector);
+      const fingerprintMatch = await pickMatching(fingerprintLocator);
+      if (fingerprintMatch) {
+        return fingerprintMatch;
+      }
+    }
+
+    // Last resort: fall back to first match from the best available selector.
+    if (await xpathLocator.count()) {
+      return xpathLocator.first();
+    }
+    if (await cssLocator.count()) {
+      return cssLocator.first();
+    }
+    if (fingerprintLocator && (await fingerprintLocator.count())) {
+      return fingerprintLocator.first();
+    }
+
+    throw new Error(
+      `Unable to resolve element for ref ${stored.ref}. Call getState() again to refresh element refs.`,
+    );
+  }
+
+  /**
+   * Get all stored refs
+   */
+  getAllRefs(): StoredElementRef[] {
+    return Array.from(this.refMap.values());
+  }
+}

package/src/state.ts

@@ -1,4 +1,5 @@
 import type { BrowserContext, Page } from "playwright";
+import type { ElementRefStore, ElementSelectors } from "./ref-store";
 import type {
   BrowserState,
   GetStateOptions,
@@ -31,7 +32,7 @@ const INTERACTIVE_SELECTORS = [
   "[tabindex]",
 ].join(", ");
 
-interface ElementInfo {
+interface RawElementInfo {
   tag: string;
   role: string;
   name: string;
@@ -40,23 +41,119 @@ interface ElementInfo {
   enabled: boolean;
   attributes: Record<string, string>;
   boundingBox: { x: number; y: number; width: number; height: number } | null;
-}
-
-interface ElementInfoWithRef extends ElementInfo {
-  ref: string;
+  // Selector info for server-side storage
+  xpath: string;
+  cssPath: string;
+  fingerprint: string | null;
+  // For generating ref
+  refBase: string;
+  // Fingerprint data for validation
+  fingerprintData: {
+    tagName: string;
+    role?: string;
+    type?: string;
+    name?: string;
+    placeholder?: string;
+  };
 }
 
 /**
- * Extract interactive elements from the page using DOM queries
- * Assumes injectElementRefs has already been called
+ * Extract interactive elements from the page WITHOUT modifying the DOM
+ * Returns raw element info including selectors for server-side ref storage
  */
-async function extractInteractiveElements(
+async function extractInteractiveElementsRaw(
   page: Page,
-): Promise<InteractiveElement[]> {
-  const elementInfos = await page.evaluate((selector) => {
-    // Only get elements that match interactive selectors and have data-ref
-    const elements = Array.from(document.querySelectorAll(selector));
-    const results: ElementInfoWithRef[] = [];
+): Promise<RawElementInfo[]> {
+  return await page.evaluate((selector) => {
+    // Helper functions (must be inside evaluate)
+    const generateXPath = (element: Element): string => {
+      const parts: string[] = [];
+      let current: Element | null = element;
+
+      while (current && current.nodeType === Node.ELEMENT_NODE) {
+        let index = 1;
+        let sibling: Element | null = current.previousElementSibling;
+
+        while (sibling) {
+          if (sibling.tagName === current.tagName) {
+            index++;
+          }
+          sibling = sibling.previousElementSibling;
+        }
+
+        const tagName = current.tagName.toLowerCase();
+        parts.unshift(`${tagName}[${index}]`);
+        current = current.parentElement;
+      }
+
+      return `/${parts.join("/")}`;
+    };
+
+    const generateCssPath = (element: Element): string => {
+      const parts: string[] = [];
+      let current: Element | null = element;
+
+      while (current && current.nodeType === Node.ELEMENT_NODE) {
+        let selector = current.tagName.toLowerCase();
+
+        const id = current.getAttribute("id");
+        if (id) {
+          try {
+            if (document.querySelectorAll(`#${CSS.escape(id)}`).length === 1) {
+              parts.unshift(`#${CSS.escape(id)}`);
+              break;
+            }
+          } catch {
+            // Invalid ID, skip
+          }
+        }
+
+        const parent = current.parentElement;
+        if (parent) {
+          const siblings = Array.from(parent.children);
+          const sameTagSiblings = siblings.filter(
+            (s) => s.tagName === current!.tagName,
+          );
+          if (sameTagSiblings.length > 1) {
+            const index = sameTagSiblings.indexOf(current) + 1;
+            selector += `:nth-of-type(${index})`;
+          }
+        }
+
+        parts.unshift(selector);
+        current = current.parentElement;
+      }
+
+      return parts.join(" > ");
+    };
+
+    const generateFingerprint = (element: Element): string | null => {
+      const tag = element.tagName.toLowerCase();
+      const attrs: string[] = [];
+
+      const type = element.getAttribute("type");
+      const name = element.getAttribute("name");
+      const placeholder = element.getAttribute("placeholder");
+      const role = element.getAttribute("role");
+      const ariaLabel = element.getAttribute("aria-label");
+      const dataTestId = element.getAttribute("data-testid");
+
+      if (dataTestId) {
+        return `[data-testid="${CSS.escape(dataTestId)}"]`;
+      }
+
+      if (type) attrs.push(`[type="${CSS.escape(type)}"]`);
+      if (name) attrs.push(`[name="${CSS.escape(name)}"]`);
+      if (placeholder) attrs.push(`[placeholder="${CSS.escape(placeholder)}"]`);
+      if (role) attrs.push(`[role="${CSS.escape(role)}"]`);
+      if (ariaLabel) attrs.push(`[aria-label="${CSS.escape(ariaLabel)}"]`);
+
+      if (attrs.length === 0) {
+        return null;
+      }
+
+      return tag + attrs.join("");
+    };
 
     const normalizeText = (value?: string | null) =>
       value?.replace(/\s+/g, " ").trim() ?? "";
@@ -109,17 +206,52 @@ async function extractInteractiveElements(
       return "";
     };
 
+    const normalizeBase = (value: string) => {
+      const trimmed = value.trim().toLowerCase();
+      const normalized = trimmed.replace(/[^a-z0-9_-]+/g, "-");
+      return normalized.length > 0 ? normalized : "element";
+    };
+
+    const getElementBase = (el: HTMLElement) => {
+      const role = el.getAttribute("role");
+      if (role) {
+        return normalizeBase(role);
+      }
+      const tag = el.tagName.toLowerCase();
+      if (tag === "a") return "link";
+      if (tag === "button") return "button";
+      if (tag === "input") {
+        const type = (el as HTMLInputElement).type;
+        if (type === "checkbox") return "checkbox";
+        if (type === "radio") return "radio";
+        if (type === "submit" || type === "button") return "button";
+        return "input";
+      }
+      if (tag === "textarea") return "textarea";
+      if (tag === "select") return "select";
+      return normalizeBase(tag);
+    };
+
+    const elements = Array.from(document.querySelectorAll(selector));
+    const results: RawElementInfo[] = [];
+
     for (const el of elements) {
       const htmlEl = el as HTMLElement;
-      const ref = htmlEl.getAttribute("data-ref");
-      if (!ref) {
+
+      // Skip hidden elements
+      const style = window.getComputedStyle(htmlEl);
+      if (style.display === "none" || style.visibility === "hidden") {
         continue;
       }
 
-      // Get bounding box
       const rect = htmlEl.getBoundingClientRect();
+      if (rect.width === 0 && rect.height === 0) {
+        const tag = htmlEl.tagName.toLowerCase();
+        if (!["input", "textarea", "select"].includes(tag)) {
+          continue;
+        }
+      }
 
-      const style = window.getComputedStyle(htmlEl);
       const isVisible =
         style.display !== "none" &&
         style.visibility !== "hidden" &&
@@ -197,12 +329,16 @@ async function extractInteractiveElements(
       if (valueText) attributes.value = valueText;
       if (isChecked) attributes.checked = "true";
 
+      // Generate selectors
+      const xpath = generateXPath(htmlEl);
+      const cssPath = generateCssPath(htmlEl);
+      const fingerprint = generateFingerprint(htmlEl);
+
       results.push({
         tag: htmlEl.tagName.toLowerCase(),
         role,
         name: name || text.slice(0, 50),
         text,
-        ref,
         visible: isVisible,
         enabled: !(htmlEl as HTMLInputElement).disabled,
         attributes,
@@ -212,24 +348,68 @@ async function extractInteractiveElements(
           width: rect.width,
           height: rect.height,
         },
+        xpath,
+        cssPath,
+        fingerprint,
+        refBase: getElementBase(htmlEl),
+        fingerprintData: {
+          tagName: htmlEl.tagName.toLowerCase(),
+          role: role || undefined,
+          type: htmlEl.getAttribute("type") || undefined,
+          name: fieldName || undefined,
+          placeholder: placeholder || undefined,
+        },
       });
     }
 
     return results;
   }, INTERACTIVE_SELECTORS);
+}
 
-  // Convert to InteractiveElement format, using ref from data-ref attribute
-  return elementInfos.map((info, index) => ({
-    index,
-    role: info.role,
-    name: info.name,
-    text: info.text,
-    ref: info.ref, // Use the actual ref from the DOM
-    visible: info.visible,
-    enabled: info.enabled,
-    boundingBox: info.boundingBox === null ? undefined : info.boundingBox,
-    attributes: info.attributes,
-  }));
+/**
+ * Extract interactive elements and store refs in the provided store
+ * Returns InteractiveElement array with assigned refs
+ */
+export async function extractInteractiveElements(
+  page: Page,
+  refStore: ElementRefStore,
+): Promise<InteractiveElement[]> {
+  const rawElements = await extractInteractiveElementsRaw(page);
+
+  // Clear and rebuild the ref store
+  refStore.clear();
+
+  // Track used refs and counters for generating unique refs
+  const counters: Record<string, number> = {};
+
+  return rawElements.map((raw, index) => {
+    // Generate unique ref
+    const base = raw.refBase;
+    const counter = counters[base] ?? 0;
+    const ref = `${base}_${counter}`;
+    counters[base] = counter + 1;
+
+    // Store selectors for later resolution
+    const selectors: ElementSelectors = {
+      xpath: raw.xpath,
+      cssPath: raw.cssPath,
+      fingerprint: raw.fingerprint ?? undefined,
+    };
+
+    refStore.set(ref, index, selectors, raw.fingerprintData);
+
+    return {
+      index,
+      role: raw.role,
+      name: raw.name,
+      text: raw.text,
+      ref,
+      visible: raw.visible,
+      enabled: raw.enabled,
+      boundingBox: raw.boundingBox === null ? undefined : raw.boundingBox,
+      attributes: raw.attributes,
+    };
+  });
 }
 
 /**
@@ -400,10 +580,12 @@ export function formatStateText(state: BrowserState): string {
 
 /**
  * Get the current state of the browser/page
+ * Now requires a refStore to store element references server-side
  */
 export async function getState(
   page: Page,
   context: BrowserContext,
+  refStore: ElementRefStore,
   options: GetStateOptions = {},
 ): Promise<BrowserState> {
   const {
@@ -421,19 +603,24 @@ export async function getState(
   // Wait for page to be stable
   await page.waitForLoadState("domcontentloaded");
 
-  // Inject refs first so extraction and targeting use same indices
-  await injectElementRefs(page);
-
-  // Extract state in parallel
-  const [url, title, elements, accessibilityTree, scrollPosition, tabs] =
-    await Promise.all([
-      page.url(),
-      page.title(),
-      includeElements ? extractInteractiveElements(page) : [],
-      includeTree ? buildAccessibilityTree(page, treeLimit) : "",
-      getScrollPosition(page),
-      getTabsInfo(context, page),
-    ]);
+  // Extract state in parallel - NO DOM MODIFICATION
+  // Always rebuild refs even if elements aren't returned.
+  const [
+    url,
+    title,
+    elementsSnapshot,
+    accessibilityTree,
+    scrollPosition,
+    tabs,
+  ] = await Promise.all([
+    page.url(),
+    page.title(),
+    extractInteractiveElements(page, refStore),
+    includeTree ? buildAccessibilityTree(page, treeLimit) : "",
+    getScrollPosition(page),
+    getTabsInfo(context, page),
+  ]);
+  const elements = includeElements ? elementsSnapshot : [];
 
   // Optional screenshot
   let screenshot: string | undefined;
@@ -509,94 +696,11 @@ function sliceTree(
 }
 
 /**
- * Inject data-ref attributes into the page for element targeting
- * Returns the number of elements tagged
+ * @deprecated No longer injects refs into DOM - refs are now stored server-side
+ * This function is kept for backwards compatibility but does nothing
  */
-export async function injectElementRefs(page: Page): Promise<number> {
-  return await page.evaluate((selector) => {
-    const elements = Array.from(document.querySelectorAll(selector));
-    const used = new Set<string>();
-    const counters: Record<string, number> = {};
-
-    const normalizeBase = (value: string) => {
-      const trimmed = value.trim().toLowerCase();
-      const normalized = trimmed.replace(/[^a-z0-9_-]+/g, "-");
-      return normalized.length > 0 ? normalized : "element";
-    };
-
-    const getElementBase = (el: HTMLElement) => {
-      const role = el.getAttribute("role");
-      if (role) {
-        return normalizeBase(role);
-      }
-      const tag = el.tagName.toLowerCase();
-      if (tag === "a") return "link";
-      if (tag === "button") return "button";
-      if (tag === "input") {
-        const type = (el as HTMLInputElement).type;
-        if (type === "checkbox") return "checkbox";
-        if (type === "radio") return "radio";
-        if (type === "submit" || type === "button") return "button";
-        return "input";
-      }
-      if (tag === "textarea") return "textarea";
-      if (tag === "select") return "select";
-      return normalizeBase(tag);
-    };
-
-    document.querySelectorAll("[data-ref]").forEach((el) => {
-      const ref = el.getAttribute("data-ref");
-      if (ref) {
-        used.add(ref);
-        const match = ref.match(/^([a-z0-9_-]+)_(\d+)$/i);
-        if (match) {
-          const base = match[1];
-          const index = Number(match[2]);
-          if (!Number.isNaN(index)) {
-            counters[base] = Math.max(counters[base] ?? 0, index + 1);
-          }
-        }
-      }
-    });
-
-    let index = 0;
-
-    for (const el of elements) {
-      const htmlEl = el as HTMLElement;
-      let ref = htmlEl.getAttribute("data-ref");
-
-      // Skip hidden elements unless they already have a stable ref.
-      const style = window.getComputedStyle(htmlEl);
-      if (!ref) {
-        if (style.display === "none" || style.visibility === "hidden") {
-          continue;
-        }
-
-        const rect = htmlEl.getBoundingClientRect();
-        if (rect.width === 0 && rect.height === 0) {
-          const tag = htmlEl.tagName.toLowerCase();
-          if (!["input", "textarea", "select"].includes(tag)) {
-            continue;
-          }
-        }
-      }
-
-      if (!ref) {
-        const base = getElementBase(htmlEl);
-        let next = counters[base] ?? 0;
-        while (used.has(`${base}_${next}`)) {
-          next++;
-        }
-        ref = `${base}_${next}`;
-        counters[base] = next + 1;
-        used.add(ref);
-        htmlEl.setAttribute("data-ref", ref);
-      }
-
-      htmlEl.setAttribute("data-index", String(index));
-      index++;
-    }
-
-    return used.size;
-  }, INTERACTIVE_SELECTORS);
+export async function injectElementRefs(_page: Page): Promise<number> {
+  // No-op: refs are now stored server-side in ElementRefStore
+  // This function is kept for API compatibility but should not be used
+  return 0;
 }

package/src/types.ts

@@ -7,6 +7,8 @@ export interface BrowserConfig {
   executablePath?: string;
   /** Prefer system Chrome/Chromium over bundled Playwright (default: true) */
   useSystemChrome?: boolean;
+  /** Allow system Chrome in headless mode on macOS (default: false) */
+  allowSystemChromeHeadless?: boolean;
   /** Viewport width (default: 1280) */
   viewportWidth?: number;
   /** Viewport height (default: 720) */