@opensteer/engine-playwright 0.8.0 → 0.8.2

package/dist/index.cjs

@@ -1259,6 +1259,282 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+// ../browser-core/src/post-load-tracker.ts
+var DEFAULT_POST_LOAD_TRACKER_QUIET_WINDOW_MS = 400;
+var DEFAULT_ACTION_BOUNDARY_POLL_INTERVAL_MS = 100;
+function isRecord(value) {
+  return typeof value === "object" && value !== null;
+}
+function readFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function readNonNegativeNumber(value) {
+  const parsed = readFiniteNumber(value);
+  return parsed === void 0 || parsed < 0 ? 0 : parsed;
+}
+function normalizePostLoadTrackerState(value) {
+  if (!isRecord(value)) {
+    return void 0;
+  }
+  const installedAt = readFiniteNumber(value.installedAt);
+  const lastMutationAt = readFiniteNumber(value.lastMutationAt);
+  const lastNetworkActivityAt = readFiniteNumber(value.lastNetworkActivityAt);
+  const now = readFiniteNumber(value.now);
+  const readyState = typeof value.readyState === "string" ? value.readyState : void 0;
+  if (installedAt === void 0 || lastMutationAt === void 0 || lastNetworkActivityAt === void 0 || now === void 0 || readyState === void 0) {
+    return void 0;
+  }
+  return {
+    installedAt,
+    lastMutationAt,
+    lastNetworkActivityAt,
+    now,
+    pendingFetches: readNonNegativeNumber(value.pendingFetches),
+    pendingTimeouts: readNonNegativeNumber(value.pendingTimeouts),
+    pendingXhrs: readNonNegativeNumber(value.pendingXhrs),
+    readyState
+  };
+}
+function buildPostLoadTrackerInstallScript() {
+  return `(() => {
+    const globalObject = globalThis;
+    if (globalObject.__opensteerActionBoundaryTrackerInstalled) {
+      return true;
+    }
+
+    const tracker = {
+      installedAt: performance.now(),
+      lastMutationAt: performance.now(),
+      lastNetworkActivityAt: performance.now(),
+      pendingFetches: 0,
+      pendingTimeouts: 0,
+      pendingXhrs: 0,
+      readyState: document.readyState,
+      timeoutIds: new Set(),
+    };
+    globalObject.__opensteerActionBoundaryTrackerInstalled = true;
+    globalObject.__opensteerActionBoundaryTracker = tracker;
+
+    const markMutation = () => {
+      tracker.lastMutationAt = performance.now();
+      tracker.readyState = document.readyState;
+    };
+    const markNetwork = () => {
+      tracker.lastNetworkActivityAt = performance.now();
+      tracker.readyState = document.readyState;
+    };
+
+    const startObserver = () => {
+      const target = document.documentElement ?? document;
+      if (!(target instanceof Node)) {
+        return;
+      }
+      const observer = new MutationObserver(markMutation);
+      observer.observe(target, {
+        subtree: true,
+        childList: true,
+        characterData: true,
+        attributes: true,
+      });
+      markMutation();
+    };
+
+    if (document.documentElement) {
+      startObserver();
+    } else {
+      document.addEventListener("DOMContentLoaded", startObserver, { once: true });
+    }
+
+    document.addEventListener("readystatechange", markMutation);
+    addEventListener("load", markMutation, { once: true });
+
+    const nativeSetTimeout = globalObject.setTimeout.bind(globalObject);
+    const nativeClearTimeout = globalObject.clearTimeout.bind(globalObject);
+    globalObject.setTimeout = function(callback, delay, ...args) {
+      tracker.pendingTimeouts += 1;
+      markNetwork();
+      let handle;
+      const wrapped =
+        typeof callback === "function"
+          ? (...callbackArgs) => {
+              if (tracker.timeoutIds.delete(handle)) {
+                tracker.pendingTimeouts = Math.max(0, tracker.pendingTimeouts - 1);
+              }
+              try {
+                return callback(...callbackArgs);
+              } finally {
+                markMutation();
+              }
+            }
+          : callback;
+      handle = nativeSetTimeout(wrapped, delay, ...args);
+      tracker.timeoutIds.add(handle);
+      return handle;
+    };
+    globalObject.clearTimeout = function(handle) {
+      if (tracker.timeoutIds.delete(handle)) {
+        tracker.pendingTimeouts = Math.max(0, tracker.pendingTimeouts - 1);
+      }
+      return nativeClearTimeout(handle);
+    };
+
+    if (typeof globalObject.fetch === "function") {
+      const nativeFetch = globalObject.fetch.bind(globalObject);
+      globalObject.fetch = (...args) => {
+        tracker.pendingFetches += 1;
+        markNetwork();
+        return nativeFetch(...args)
+          .finally(() => {
+            tracker.pendingFetches = Math.max(0, tracker.pendingFetches - 1);
+            markNetwork();
+          });
+      };
+    }
+
+    if (typeof globalObject.XMLHttpRequest === "function") {
+      const NativeXMLHttpRequest = globalObject.XMLHttpRequest;
+      const nativeSend = NativeXMLHttpRequest.prototype.send;
+      NativeXMLHttpRequest.prototype.send = function(...args) {
+        tracker.pendingXhrs += 1;
+        markNetwork();
+        const finalize = () => {
+          this.removeEventListener("loadend", finalize);
+          tracker.pendingXhrs = Math.max(0, tracker.pendingXhrs - 1);
+          markNetwork();
+        };
+        this.addEventListener("loadend", finalize, { once: true });
+        return nativeSend.apply(this, args);
+      };
+    }
+
+    return true;
+  })()`;
+}
+function buildPostLoadTrackerReadExpression() {
+  return `(() => {
+    const tracker = globalThis.__opensteerActionBoundaryTracker;
+    if (!tracker) {
+      return null;
+    }
+
+    return {
+      installedAt: Number(tracker.installedAt ?? 0),
+      lastMutationAt: Number(tracker.lastMutationAt ?? 0),
+      lastNetworkActivityAt: Number(tracker.lastNetworkActivityAt ?? 0),
+      now: Number(performance.now()),
+      pendingFetches: Number(tracker.pendingFetches ?? 0),
+      pendingTimeouts: Number(tracker.pendingTimeouts ?? 0),
+      pendingXhrs: Number(tracker.pendingXhrs ?? 0),
+      readyState: String(document.readyState),
+    };
+  })()`;
+}
+function postLoadTrackerIsSettled(tracker, quietWindowMs = DEFAULT_POST_LOAD_TRACKER_QUIET_WINDOW_MS) {
+  if (!tracker) {
+    return false;
+  }
+  if (tracker.readyState !== "complete") {
+    return false;
+  }
+  if (tracker.pendingFetches > 0 || tracker.pendingTimeouts > 0 || tracker.pendingXhrs > 0) {
+    return false;
+  }
+  const lastActivityAt = Math.max(
+    tracker.installedAt,
+    tracker.lastMutationAt,
+    tracker.lastNetworkActivityAt
+  );
+  return tracker.now - lastActivityAt >= quietWindowMs;
+}
+
+// ../browser-core/src/action-boundary.ts
+var CROSS_DOCUMENT_INTERACTION_TIMEOUT_MS = 3e4;
+var CROSS_DOCUMENT_DETECTION_WINDOW_MS = 500;
+async function waitForActionBoundary(input) {
+  if (input.timeoutMs <= 0) {
+    return {
+      trigger: "dom-action",
+      crossDocument: false,
+      bootstrapSettled: false,
+      timedOutPhase: "bootstrap"
+    };
+  }
+  const deadline = Date.now() + input.timeoutMs;
+  const crossDocumentDetectionDeadline = input.snapshot === void 0 ? void 0 : Math.min(deadline, Date.now() + CROSS_DOCUMENT_DETECTION_WINDOW_MS);
+  const pollIntervalMs = input.pollIntervalMs ?? DEFAULT_ACTION_BOUNDARY_POLL_INTERVAL_MS;
+  let trigger = "dom-action";
+  let crossDocument = false;
+  let waitedForNavigationContentLoaded = false;
+  while (Date.now() < deadline) {
+    input.throwBackgroundError();
+    if (input.isPageClosed()) {
+      return {
+        trigger,
+        crossDocument,
+        bootstrapSettled: true
+      };
+    }
+    if (input.signal?.aborted) {
+      if (isTimeoutAbort(input.signal.reason) && Date.now() >= deadline) {
+        return {
+          trigger,
+          crossDocument,
+          bootstrapSettled: false,
+          timedOutPhase: "bootstrap"
+        };
+      }
+      throw abortError(input.signal);
+    }
+    const currentDocumentRef = input.getCurrentMainFrameDocumentRef();
+    if (input.snapshot !== void 0 && currentDocumentRef !== void 0 && currentDocumentRef !== input.snapshot.documentRef) {
+      trigger = "navigation";
+      crossDocument = true;
+      if (!waitedForNavigationContentLoaded) {
+        waitedForNavigationContentLoaded = true;
+        const remaining = Math.max(0, deadline - Date.now());
+        if (remaining > 0) {
+          await input.waitForNavigationContentLoaded(remaining);
+        }
+      }
+    }
+    if (!crossDocument && crossDocumentDetectionDeadline !== void 0 && Date.now() >= crossDocumentDetectionDeadline) {
+      return {
+        trigger,
+        crossDocument,
+        bootstrapSettled: true
+      };
+    }
+    if (crossDocument && postLoadTrackerIsSettled(await input.readTrackerState())) {
+      return {
+        trigger,
+        crossDocument,
+        bootstrapSettled: true
+      };
+    }
+    await delay(Math.min(pollIntervalMs, Math.max(0, deadline - Date.now())));
+  }
+  return {
+    trigger,
+    crossDocument,
+    bootstrapSettled: false,
+    timedOutPhase: "bootstrap"
+  };
+}
+function abortError(signal) {
+  return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
+}
+async function delay(ms) {
+  if (ms <= 0) {
+    return;
+  }
+  await new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+function isTimeoutAbort(reason) {
+  return typeof reason === "object" && reason !== null && "code" in reason && reason.code === "timeout";
+}
+
 // ../protocol/src/computer-use-bridge.ts
 var OPENSTEER_COMPUTER_USE_BRIDGE_SYMBOL = /* @__PURE__ */ Symbol.for("@opensteer/computer-use-bridge");
 
@@ -1743,6 +2019,7 @@ function createPlaywrightComputerUseBridge(context) {
       const startedAt = Date.now();
       const actionController = context.resolveController(input.pageRef);
       const action = input.action;
+      let boundary;
       let actionMs = 0;
       let waitMs = 0;
       const actionStartedAt = Date.now();
@@ -1808,7 +2085,12 @@ function createPlaywrightComputerUseBridge(context) {
       await context.flushPendingPageTasks(actionController.sessionRef);
       if (action.type !== "screenshot" && action.type !== "wait") {
         const waitStartedAt = Date.now();
-        await input.policySettle(actionController.pageRef);
+        boundary = await context.settleActionBoundary(actionController, {
+          signal: input.signal,
+          ...input.snapshot === void 0 ? {} : { snapshot: input.snapshot },
+          remainingMs: input.remainingMs,
+          policySettle: input.policySettle
+        });
         waitMs = Date.now() - waitStartedAt;
       } else if (action.type === "wait") {
         waitMs = actionMs;
@@ -1820,7 +2102,11 @@ function createPlaywrightComputerUseBridge(context) {
       let resultController = context.resolveController(resultPageRef);
       if (action.type !== "screenshot" && action.type !== "wait" && resultController.pageRef !== actionController.pageRef) {
         const popupWaitStartedAt = Date.now();
-        await input.policySettle(resultController.pageRef);
+        await context.settleActionBoundary(resultController, {
+          signal: input.signal,
+          remainingMs: input.remainingMs,
+          policySettle: input.policySettle
+        });
         waitMs += Date.now() - popupWaitStartedAt;
         await context.flushPendingPageTasks(actionController.sessionRef);
         resultController = context.resolveController(resultController.pageRef);
@@ -1844,7 +2130,8 @@ function createPlaywrightComputerUseBridge(context) {
           actionMs,
           waitMs,
           totalMs: Date.now() - startedAt
-        }
+        },
+        ...boundary === void 0 ? {} : { boundary }
       };
     }
   };
@@ -2877,10 +3164,12 @@ function createPlaywrightDomActionBridge(context) {
     },
     async finalizeDomAction(pageRef, options) {
       const controller = context.resolveController(pageRef);
-      await context.flushPendingPageTasks(controller.sessionRef);
-      await options.policySettle(pageRef);
-      await context.flushPendingPageTasks(controller.sessionRef);
-      await context.flushDomUpdateTask(controller);
+      return context.settleActionBoundary(controller, {
+        signal: options.signal,
+        ...options.snapshot === void 0 ? {} : { snapshot: options.snapshot },
+        remainingMs: options.remainingMs,
+        policySettle: options.policySettle
+      });
     }
   };
 }
@@ -3148,6 +3437,121 @@ async function releaseObject(controller, objectId) {
   await controller.cdp.send("Runtime.releaseObject", { objectId }).catch(() => void 0);
 }
 
+// src/action-settle.ts
+var DEFAULT_PLAYWRIGHT_ACTION_SETTLE_TIMEOUT_MS = CROSS_DOCUMENT_INTERACTION_TIMEOUT_MS;
+function clampPlaywrightActionSettleTimeout(timeoutMs) {
+  if (timeoutMs === void 0) {
+    return DEFAULT_PLAYWRIGHT_ACTION_SETTLE_TIMEOUT_MS;
+  }
+  return Math.max(0, Math.min(DEFAULT_PLAYWRIGHT_ACTION_SETTLE_TIMEOUT_MS, timeoutMs));
+}
+function createPlaywrightActionSettler(context) {
+  const installScript = buildPostLoadTrackerInstallScript();
+  const readExpression = buildPostLoadTrackerReadExpression();
+  async function installTracker(controller) {
+    if (!controller.settleTrackerRegistered) {
+      await controller.page.addInitScript(installScript);
+      controller.settleTrackerRegistered = true;
+    }
+    try {
+      await controller.cdp.send("Runtime.evaluate", {
+        expression: installScript,
+        returnByValue: true,
+        awaitPromise: true
+      });
+    } catch (error) {
+      if (controller.lifecycleState === "closed" || isContextClosedError(error)) {
+        return;
+      }
+      throw normalizePlaywrightError(error, controller.pageRef);
+    }
+  }
+  async function readTrackerState(controller) {
+    try {
+      const evaluated = await controller.cdp.send("Runtime.evaluate", {
+        expression: readExpression,
+        returnByValue: true,
+        awaitPromise: true
+      });
+      return normalizePostLoadTrackerState(evaluated.result?.value);
+    } catch (error) {
+      if (isIgnorableTrackerReadError(error)) {
+        return void 0;
+      }
+      throw normalizePlaywrightError(error, controller.pageRef);
+    }
+  }
+  async function settle(options) {
+    const { controller, timeoutMs, signal, snapshot, policySettle } = options;
+    if (timeoutMs <= 0) {
+      return {
+        trigger: "dom-action",
+        crossDocument: false,
+        bootstrapSettled: false,
+        timedOutPhase: "bootstrap"
+      };
+    }
+    await context.flushPendingPageTasks(controller.sessionRef);
+    let boundary;
+    if (snapshot === void 0) {
+      if (policySettle) {
+        if (signal?.aborted) {
+          throw signal.reason ?? abortError2();
+        }
+        await policySettle(controller.pageRef, "dom-action");
+      }
+      boundary = {
+        trigger: "dom-action",
+        crossDocument: false,
+        bootstrapSettled: true
+      };
+    } else {
+      await installTracker(controller);
+      boundary = await waitForActionBoundary({
+        timeoutMs,
+        ...signal === void 0 ? {} : { signal },
+        snapshot,
+        getCurrentMainFrameDocumentRef: () => context.getMainFrameDocumentRef(controller),
+        waitForNavigationContentLoaded: async (remainingMs) => {
+          try {
+            await controller.page.waitForLoadState("domcontentloaded", {
+              timeout: remainingMs
+            });
+          } catch (error) {
+            if (controller.lifecycleState === "closed" || isContextClosedError(error)) {
+              return;
+            }
+            throw normalizePlaywrightError(error, controller.pageRef);
+          }
+        },
+        readTrackerState: () => readTrackerState(controller),
+        throwBackgroundError: () => context.throwBackgroundError(controller),
+        isPageClosed: () => controller.lifecycleState === "closed"
+      });
+      if (policySettle) {
+        await policySettle(controller.pageRef, boundary.trigger);
+      }
+    }
+    await context.flushPendingPageTasks(controller.sessionRef);
+    if (controller.lifecycleState !== "closed") {
+      await context.flushDomUpdateTask(controller);
+    }
+    return boundary;
+  }
+  return {
+    installTracker,
+    settle
+  };
+}
+function abortError2() {
+  return new DOMException("The operation was aborted", "AbortError");
+}
+function isIgnorableTrackerReadError(error) {
+  return isContextClosedError(error) || error instanceof Error && /Execution context was destroyed|Cannot find context|Inspected target navigated or closed/i.test(
+    error.message
+  );
+}
+
 // src/storage-capture.ts
 var ACTIVATION_PATH = "/__opensteer_storage_capture__";
 var ACTIVATION_TIMEOUT_MS = 15e3;
@@ -3354,6 +3758,12 @@ var PlaywrightBrowserCoreEngine = class _PlaywrightBrowserCoreEngine {
   pageByPlaywrightPage = /* @__PURE__ */ new WeakMap();
   pendingPopupOpeners = /* @__PURE__ */ new WeakMap();
   preassignedPopupPageRefs = /* @__PURE__ */ new WeakMap();
+  actionSettler = createPlaywrightActionSettler({
+    flushPendingPageTasks: (sessionRef) => this.flushPendingPageTasks(sessionRef),
+    flushDomUpdateTask: (controller) => this.flushDomUpdateTask(controller),
+    getMainFrameDocumentRef: (controller) => controller.mainFrameRef === void 0 ? void 0 : this.frames.get(controller.mainFrameRef)?.currentDocument.documentRef,
+    throwBackgroundError: (controller) => this.throwBackgroundError(controller)
+  });
   pageCounter = 0;
   frameCounter = 0;
   documentCounter = 0;
@@ -3409,6 +3819,13 @@ var PlaywrightBrowserCoreEngine = class _PlaywrightBrowserCoreEngine {
       resolveController: (pageRef) => this.requirePage(pageRef),
       flushPendingPageTasks: (sessionRef) => this.flushPendingPageTasks(sessionRef),
       flushDomUpdateTask: (controller) => this.flushDomUpdateTask(controller),
+      settleActionBoundary: (controller, options) => this.actionSettler.settle({
+        controller,
+        timeoutMs: clampPlaywrightActionSettleTimeout(options.remainingMs()),
+        ...options.signal === void 0 ? {} : { signal: options.signal },
+        ...options.snapshot === void 0 ? {} : { snapshot: options.snapshot },
+        ...options.policySettle === void 0 ? {} : { policySettle: options.policySettle }
+      }),
       requireMainFrame: (controller) => this.requireMainFrame(controller),
       drainQueuedEvents: (pageRef) => this.drainQueuedEvents(pageRef),
       withModifiers: (page, modifiers, action) => this.withModifiers(page, modifiers, action)
@@ -3420,6 +3837,13 @@ var PlaywrightBrowserCoreEngine = class _PlaywrightBrowserCoreEngine {
       resolveController: (pageRef) => this.requirePage(pageRef),
       flushPendingPageTasks: (sessionRef) => this.flushPendingPageTasks(sessionRef),
       flushDomUpdateTask: (controller) => this.flushDomUpdateTask(controller),
+      settleActionBoundary: (controller, options) => this.actionSettler.settle({
+        controller,
+        timeoutMs: clampPlaywrightActionSettleTimeout(options.remainingMs()),
+        ...options.signal === void 0 ? {} : { signal: options.signal },
+        ...options.snapshot === void 0 ? {} : { snapshot: options.snapshot },
+        ...options.policySettle === void 0 ? {} : { policySettle: options.policySettle }
+      }),
       locateBackendNode: (document, backendNodeId) => createNodeLocator(
         document.documentRef,
         document.documentEpoch,
@@ -4514,6 +4938,7 @@ var PlaywrightBrowserCoreEngine = class _PlaywrightBrowserCoreEngine {
       backgroundTasks: /* @__PURE__ */ new Set(),
       domUpdateTask: void 0,
       backgroundError: void 0,
+      settleTrackerRegistered: false,
       openerPageRef: void 0,
       mainFrameRef: void 0,
       lifecycleState: "open",
@@ -4530,6 +4955,7 @@ var PlaywrightBrowserCoreEngine = class _PlaywrightBrowserCoreEngine {
     await cdp.send("DOM.enable", { includeWhitespace: "none" });
     await cdp.send("DOMStorage.enable");
     await cdp.send("DOM.getDocument", { depth: 0 });
+    await this.actionSettler.installTracker(controller);
     cdp.on(
       "Page.frameAttached",
       (payload) => this.runControllerEvent(