@intuned/browser-dev 0.1.9-dev.0 → 0.1.10-dev.0

package/dist/common/extendedTest.js

@@ -47,26 +47,26 @@ Object.defineProperty(exports, "vi", {
     return _vitest.vi;
   }
 });
+var _playwright = require("playwright");
 var dotenv = _interopRequireWildcard(require("dotenv"));
 var _vitest = require("vitest");
 var _nanoid = require("nanoid");
 var _extractionHelpers = require("./extractionHelpers");
+var _path = _interopRequireDefault(require("path"));
 var _script = require("./script");
+var _os = _interopRequireDefault(require("os"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
 dotenv.config();
 const loadRuntime = async () => {
   try {
     const runtime = await Promise.resolve().then(() => _interopRequireWildcard(require("@intuned/runtime")));
     return {
-      withPlaywrightContext: runtime.withPlaywrightContext,
-      runWithContext: runtime.runWithContext,
-      getDownloadDirectoryPath: runtime.getDownloadDirectoryPath
+      runWithContext: runtime.runWithContext
     };
   } catch {
     return {
-      withPlaywrightContext: null,
-      runWithContext: null,
-      getDownloadDirectoryPath: null
+      runWithContext: null
     };
   }
 };
@@ -80,49 +80,57 @@ const withPlaywright = (name, testFn, {
 }) => {
   return async () => {
     const runtime = await loadRuntime();
-    if (!runtime.runWithContext || !runtime.withPlaywrightContext || !runtime.getDownloadDirectoryPath) {
+    if (!runtime.runWithContext) {
       throw new Error("@intuned/runtime is required for running tests. Please install it as a dependency.");
     }
+    const downloadsPath = _path.default.join(_os.default.tmpdir(), `downloads-${(0, _nanoid.nanoid)()}`);
     return runtime.runWithContext({
       runId: runId ?? (0, _nanoid.nanoid)(),
       extendedPayloads: [],
       runEnvironment: "IDE",
       jobId,
       jobRunId,
-      queueId
+      queueId,
+      headless: headless ?? true,
+      downloadsPath
     }, async () => {
-      const downloadsPath = runtime.getDownloadDirectoryPath();
-      await runtime.withPlaywrightContext({
-        headless: headless ?? true,
-        downloadsPath
-      }, async (context, page) => {
-        context.setDefaultTimeout(10000);
-        try {
-          await context.addInitScript({
-            content: _script.BROWSER_SCRIPT
-          });
-          for (const page of context.pages()) {
-            await page.evaluate(_script.BROWSER_SCRIPT);
-          }
-        } catch (error) {
-          console.error("Error reading browser scripts:", error);
-        }
-        if (trace) {
-          await context.tracing.start({
-            screenshots: true
-          });
+      const browser = await _playwright.chromium.launch({
+        headless: headless ?? true
+      });
+      const context = await browser.newContext({
+        acceptDownloads: true
+      });
+      const page = await context.newPage();
+      context.setDefaultTimeout(10000);
+      try {
+        await context.addInitScript({
+          content: _script.BROWSER_SCRIPT
+        });
+        for (const p of context.pages()) {
+          await p.evaluate(_script.BROWSER_SCRIPT);
         }
+      } catch (error) {
+        console.error("Error reading browser scripts:", error);
+      }
+      if (trace) {
+        await context.tracing.start({
+          screenshots: true
+        });
+      }
+      try {
         await testFn({
           context: context,
           page
         });
+      } finally {
         if (trace) {
           await context.tracing.stop({
             path: `./reports/traces/${name}.zip`
           });
         }
-        return {};
-      });
+        await context.close();
+        await browser.close();
+      }
     });
   };
 };

package/dist/common/frame_utils/frameTree.js

@@ -0,0 +1,116 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.FrameTree = exports.FrameNode = void 0;
+var _Logger = require("../Logger");
+var _checkFrameAllowsAsyncScripts = require("./checkFrameAllowsAsyncScripts");
+var _utils = require("./utils");
+class FrameNode {
+  constructor(frame, frameElement, allowsAsyncScripts, nestedIframes) {
+    this.frame = frame;
+    this.frameElement = frameElement;
+    this.allowsAsyncScripts = allowsAsyncScripts;
+    this._nestedIframes = nestedIframes;
+  }
+  *nestedIframes() {
+    for (const node of this._nestedIframes) {
+      if (node.frame.isDetached()) {
+        _Logger.logger.warn(`Detached frame skipped: ${node.frame}`);
+        continue;
+      }
+      yield node;
+    }
+  }
+}
+exports.FrameNode = FrameNode;
+class FrameTree {
+  constructor(root, tree) {
+    this.root = root;
+    this._tree = tree;
+  }
+  *tree() {
+    for (const node of this._tree) {
+      if (node.frame.isDetached()) {
+        _Logger.logger.warn(`Detached frame skipped: ${node.frame}`);
+        continue;
+      }
+      yield node;
+    }
+  }
+  *nodes() {
+    for (const node of this.tree()) {
+      yield node;
+      yield* this._traverseNested(node);
+    }
+  }
+  *_traverseNested(node) {
+    for (const child of node.nestedIframes()) {
+      yield child;
+      yield* this._traverseNested(child);
+    }
+  }
+  static async fromRoot(root, iframeTimeout = 10.0, skipUseless = true) {
+    const processed = new Set();
+    const tree = await processFrameRecursive(root, processed, iframeTimeout, skipUseless);
+    return new FrameTree(root, tree);
+  }
+}
+exports.FrameTree = FrameTree;
+async function processFrameRecursive(root, processedFrames, iframeTimeout, skipUseless) {
+  const isFrame = "frameElement" in root && typeof root.frameElement === "function";
+  if (isFrame && processedFrames.has(root)) {
+    return [];
+  }
+  if (isFrame) {
+    processedFrames.add(root);
+  }
+  const iframeNodes = [];
+  try {
+    let iframeHandles;
+    try {
+      iframeHandles = await Promise.race([(0, _utils.findTopLevelIframeElements)(root, skipUseless), new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), iframeTimeout * 1000))]);
+    } catch (error) {
+      _Logger.logger.error("Timeout finding iframe elements, skipping");
+      return [];
+    }
+    if (!iframeHandles || iframeHandles.length === 0) {
+      return [];
+    }
+    _Logger.logger.debug(`Found ${iframeHandles.length} iframe elements to process`);
+    for (const iframeHandle of iframeHandles) {
+      try {
+        const iframeNode = await Promise.race([processSingleIframe(iframeHandle, processedFrames, iframeTimeout, skipUseless), new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), iframeTimeout * 1000))]);
+        if (iframeNode !== null) {
+          iframeNodes.push(iframeNode);
+        }
+      } catch (error) {
+        _Logger.logger.error("Timeout processing iframe, skipping");
+        continue;
+      }
+    }
+  } catch (error) {
+    _Logger.logger.error(`Error processing frames in context: ${error}`);
+  }
+  return iframeNodes;
+}
+async function processSingleIframe(iframeHandle, processedFrames, iframeTimeout, skipUseless) {
+  let contentFrame;
+  try {
+    contentFrame = await Promise.race([iframeHandle.contentFrame(), new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout accessing contentFrame")), iframeTimeout * 1000))]);
+  } catch (error) {
+    _Logger.logger.error("Timeout or error accessing content_frame for iframe, skipping");
+    return null;
+  }
+  if (!contentFrame) {
+    _Logger.logger.error("Could not access content_frame for iframe");
+    return null;
+  }
+  if (processedFrames.has(contentFrame)) {
+    return null;
+  }
+  const allowsAsyncScripts = await (0, _checkFrameAllowsAsyncScripts.checkFrameAllowsAsyncScripts)(iframeHandle);
+  const nestedIframes = await processFrameRecursive(contentFrame, processedFrames, iframeTimeout, skipUseless);
+  return new FrameNode(contentFrame, iframeHandle, allowsAsyncScripts, nestedIframes);
+}

package/dist/common/frame_utils/getContentWithNestedIframes.js

@@ -0,0 +1,13 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getContentWithNestedIframes = getContentWithNestedIframes;
+var _frameTree = require("./frameTree");
+var _stitchIframe = require("./stitchIframe");
+async function getContentWithNestedIframes(root, iframeTimeout = 10.0, htmlContentExtractor) {
+  const iframeTree = await _frameTree.FrameTree.fromRoot(root, iframeTimeout, true);
+  const soup = await (0, _stitchIframe.stitchIframeContents)(iframeTree, htmlContentExtractor);
+  return soup.toString();
+}

package/dist/common/frame_utils/index.js

@@ -0,0 +1,95 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "ALL_IFRAMES_CSS_SELECTOR", {
+  enumerable: true,
+  get: function () {
+    return _utils.ALL_IFRAMES_CSS_SELECTOR;
+  }
+});
+Object.defineProperty(exports, "FrameNode", {
+  enumerable: true,
+  get: function () {
+    return _frameTree.FrameNode;
+  }
+});
+Object.defineProperty(exports, "FrameTree", {
+  enumerable: true,
+  get: function () {
+    return _frameTree.FrameTree;
+  }
+});
+Object.defineProperty(exports, "HtmlContentExtractor", {
+  enumerable: true,
+  get: function () {
+    return _getContentWithNestedIframes.HtmlContentExtractor;
+  }
+});
+Object.defineProperty(exports, "IFRAME_CAPTCHA_SRC_PATTERNS", {
+  enumerable: true,
+  get: function () {
+    return _utils.IFRAME_CAPTCHA_SRC_PATTERNS;
+  }
+});
+Object.defineProperty(exports, "IFRAME_PIXEL_SRC_DOMAINS", {
+  enumerable: true,
+  get: function () {
+    return _utils.IFRAME_PIXEL_SRC_DOMAINS;
+  }
+});
+Object.defineProperty(exports, "IFRAME_REPLACEMENT_TAG", {
+  enumerable: true,
+  get: function () {
+    return _utils.IFRAME_REPLACEMENT_TAG;
+  }
+});
+Object.defineProperty(exports, "IFRAME_SRC_ATTRS", {
+  enumerable: true,
+  get: function () {
+    return _utils.IFRAME_SRC_ATTRS;
+  }
+});
+Object.defineProperty(exports, "IFRAME_TAGS", {
+  enumerable: true,
+  get: function () {
+    return _utils.IFRAME_TAGS;
+  }
+});
+Object.defineProperty(exports, "checkFrameAllowsAsyncScripts", {
+  enumerable: true,
+  get: function () {
+    return _checkFrameAllowsAsyncScripts.checkFrameAllowsAsyncScripts;
+  }
+});
+Object.defineProperty(exports, "findTopLevelIframeElements", {
+  enumerable: true,
+  get: function () {
+    return _utils.findTopLevelIframeElements;
+  }
+});
+Object.defineProperty(exports, "getContainerFrame", {
+  enumerable: true,
+  get: function () {
+    return _getContainerFrame.getContainerFrame;
+  }
+});
+Object.defineProperty(exports, "getContentWithNestedIframes", {
+  enumerable: true,
+  get: function () {
+    return _getContentWithNestedIframes.getContentWithNestedIframes;
+  }
+});
+Object.defineProperty(exports, "stitchIframeContents", {
+  enumerable: true,
+  get: function () {
+    return _stitchIframe.stitchIframeContents;
+  }
+});
+var _utils = require("./utils");
+var _frameTree = require("./frameTree");
+var _stitchIframe = require("./stitchIframe");
+var _getContentWithNestedIframes = require("./getContentWithNestedIframes");
+var _getContainerFrame = require("./getContainerFrame");
+var _checkFrameAllowsAsyncScripts = require("./checkFrameAllowsAsyncScripts");

package/dist/common/frame_utils/stitchIframe.js

@@ -0,0 +1,105 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.stitchIframeContents = stitchIframeContents;
+var _nodeHtmlParser = require("node-html-parser");
+var _uuid = require("uuid");
+var _Logger = require("../Logger");
+var _utils = require("./utils");
+async function stitchIframeContents(iframeTree, extractHtml) {
+  const defaultExtractHtml = async root => {
+    if ("content" in root && typeof root.content === "function") {
+      return await root.content();
+    } else {
+      return await root.evaluate(el => el.outerHTML);
+    }
+  };
+  const extract = extractHtml || defaultExtractHtml;
+  const frameToMarker = new Map();
+  for (const node of iframeTree.nodes()) {
+    try {
+      const markerId = `iframe-${(0, _uuid.v4)().substring(0, 8)}`;
+      await node.frameElement.evaluate((el, marker) => el.setAttribute("data-iframe-marker", marker), markerId);
+      frameToMarker.set(node.frame, markerId);
+    } catch (error) {
+      _Logger.logger.warn(`Error adding marker to iframe: ${error}`);
+    }
+  }
+  try {
+    const root = iframeTree.root;
+    const rootContent = await extract(root);
+    const frameContents = new Map();
+    for (const node of iframeTree.nodes()) {
+      try {
+        frameContents.set(node.frame, await extract(node.frame));
+      } catch (error) {
+        _Logger.logger.warn(`Error extracting content from frame: ${error}`);
+      }
+    }
+    const mainSoup = (0, _nodeHtmlParser.parse)(rootContent);
+    for (const iframeNode of iframeTree.tree()) {
+      stitchIframeRecursive(mainSoup, iframeNode, frameContents, frameToMarker);
+    }
+    return mainSoup;
+  } finally {
+    for (const node of iframeTree.nodes()) {
+      try {
+        await node.frameElement.evaluate(el => el.removeAttribute("data-iframe-marker"));
+      } catch (error) {
+        _Logger.logger.warn(`Error removing marker from iframe: ${error}`);
+      }
+    }
+  }
+}
+function stitchIframeRecursive(parentSoup, iframeNode, frameContents, frameToMarker) {
+  const content = frameContents.get(iframeNode.frame);
+  if (content === undefined) {
+    _Logger.logger.warn(`No content found for frame ${iframeNode.frame}`);
+    return;
+  }
+  const markerId = frameToMarker.get(iframeNode.frame);
+  if (!markerId) {
+    _Logger.logger.warn(`No marker found for frame ${iframeNode.frame}`);
+    return;
+  }
+  let iframeElement = null;
+  for (const tag of _utils.IFRAME_TAGS) {
+    const element = parentSoup.querySelector(`${tag}[data-iframe-marker="${markerId}"]`);
+    if (element) {
+      iframeElement = element;
+      break;
+    }
+  }
+  if (!iframeElement) {
+    _Logger.logger.warn(`Could not find iframe element with marker ${markerId} in soup`);
+    return;
+  }
+  const frameSoup = (0, _nodeHtmlParser.parse)(content);
+  const htmlElement = frameSoup.querySelector("html");
+  if (!htmlElement) {
+    _Logger.logger.warn(`No html element found in iframe ${markerId}`);
+    return;
+  }
+  for (const nestedNode of iframeNode.nestedIframes()) {
+    stitchIframeRecursive(frameSoup, nestedNode, frameContents, frameToMarker);
+  }
+  if (iframeElement.hasAttribute("data-iframe-marker")) {
+    iframeElement.removeAttribute("data-iframe-marker");
+  }
+  const bodyElement = frameSoup.querySelector("body");
+  iframeElement.innerHTML = "";
+  if (bodyElement) {
+    const children = Array.from(bodyElement.childNodes);
+    for (const child of children) {
+      iframeElement.appendChild(child);
+    }
+  } else {
+    iframeElement.appendChild(htmlElement);
+  }
+  const originalTagName = iframeElement.rawTagName.toLowerCase();
+  if (originalTagName in _utils.IFRAME_REPLACEMENT_TAG) {
+    iframeElement.rawTagName = _utils.IFRAME_REPLACEMENT_TAG[originalTagName];
+  }
+}

package/dist/{helpers → common}/frame_utils/tests/testFindAllIframes.spec.js

@@ -2,8 +2,8 @@
 
 var _extendedTest = require("../../../common/extendedTest");
 var _playwright = require("playwright");
-var _findAllIframes = require("../findAllIframes");
-(0, _extendedTest.describe)("Test findAllIframes", () => {
+var _frameTree = require("../frameTree");
+(0, _extendedTest.describe)("Test FrameTree", () => {
   let browser;
   let page;
   (0, _extendedTest.beforeAll)(async () => {
@@ -39,12 +39,13 @@ var _findAllIframes = require("../findAllIframes");
       </html>`, {
       waitUntil: "domcontentloaded"
     });
-    const iframeNodes = await (0, _findAllIframes.findAllIframes)(page);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page);
+    const iframeNodes = Array.from(iframeTree.tree());
     (0, _extendedTest.expect)(iframeNodes.length).toBe(2);
     (0, _extendedTest.expect)(iframeNodes[0].frame).toBeDefined();
     (0, _extendedTest.expect)(iframeNodes[1].frame).toBeDefined();
-    (0, _extendedTest.expect)(iframeNodes[0].nestedIframes.length).toBe(0);
-    (0, _extendedTest.expect)(iframeNodes[1].nestedIframes.length).toBe(0);
+    (0, _extendedTest.expect)(Array.from(iframeNodes[0].nestedIframes()).length).toBe(0);
+    (0, _extendedTest.expect)(Array.from(iframeNodes[1].nestedIframes()).length).toBe(0);
   });
   (0, _extendedTest.test)("should find nested iframes", async () => {
     await page.goto(`data:text/html,
@@ -60,10 +61,12 @@ var _findAllIframes = require("../findAllIframes");
       </html>`, {
       waitUntil: "domcontentloaded"
     });
-    const iframeNodes = await (0, _findAllIframes.findAllIframes)(page);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page);
+    const iframeNodes = Array.from(iframeTree.tree());
     (0, _extendedTest.expect)(iframeNodes.length).toBe(1);
-    (0, _extendedTest.expect)(iframeNodes[0].nestedIframes.length).toBe(1);
-    (0, _extendedTest.expect)(iframeNodes[0].nestedIframes[0].nestedIframes.length).toBe(0);
+    const nestedIframes = Array.from(iframeNodes[0].nestedIframes());
+    (0, _extendedTest.expect)(nestedIframes.length).toBe(1);
+    (0, _extendedTest.expect)(Array.from(nestedIframes[0].nestedIframes()).length).toBe(0);
   });
   (0, _extendedTest.test)("should handle page with no iframes", async () => {
     await page.goto(`data:text/html,
@@ -75,7 +78,8 @@ var _findAllIframes = require("../findAllIframes");
       </html>`, {
       waitUntil: "domcontentloaded"
     });
-    const iframeNodes = await (0, _findAllIframes.findAllIframes)(page);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page);
+    const iframeNodes = Array.from(iframeTree.tree());
     (0, _extendedTest.expect)(iframeNodes.length).toBe(0);
   });
   (0, _extendedTest.test)("should handle problematic iframe sources", async () => {
@@ -104,7 +108,8 @@ var _findAllIframes = require("../findAllIframes");
       timeout: 5000
     });
     const startTime = Date.now();
-    const iframeNodes = await (0, _findAllIframes.findAllIframes)(page, 2000);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page, 2.0);
+    const iframeNodes = Array.from(iframeTree.tree());
     const elapsedTime = Date.now() - startTime;
     (0, _extendedTest.expect)(elapsedTime).toBeLessThan(10000);
     (0, _extendedTest.expect)(iframeNodes.length).toBeLessThan(3);
@@ -123,7 +128,8 @@ var _findAllIframes = require("../findAllIframes");
       </html>`, {
       waitUntil: "domcontentloaded"
     });
-    const iframeNodes = await (0, _findAllIframes.findAllIframes)(page);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page);
+    const iframeNodes = Array.from(iframeTree.tree());
     (0, _extendedTest.expect)(iframeNodes.length).toBe(1);
   });
   (0, _extendedTest.test)("should handle legacy frame elements", async () => {
@@ -141,13 +147,14 @@ var _findAllIframes = require("../findAllIframes");
       </html>`, {
       waitUntil: "domcontentloaded"
     });
-    const iframeNodes = await (0, _findAllIframes.findAllIframes)(page);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page);
+    const iframeNodes = Array.from(iframeTree.tree());
     (0, _extendedTest.expect)(iframeNodes.length).toBeGreaterThanOrEqual(0);
     for (const node of iframeNodes) {
       (0, _extendedTest.expect)(node.frame).toBeDefined();
     }
   });
-  (0, _extendedTest.test)("findAllIframesList should return flat list", async () => {
+  (0, _extendedTest.test)("nodes() should return flat list", async () => {
     await page.goto(`data:text/html,
       <html>
         <body>
@@ -161,7 +168,8 @@ var _findAllIframes = require("../findAllIframes");
       </html>`, {
       waitUntil: "domcontentloaded"
     });
-    const iframeList = await (0, _findAllIframes.findAllIframesList)(page);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page);
+    const iframeList = Array.from(iframeTree.nodes());
     (0, _extendedTest.expect)(iframeList.length).toBe(2);
     for (const node of iframeList) {
       (0, _extendedTest.expect)(node.frame).toBeDefined();
@@ -188,7 +196,8 @@ var _findAllIframes = require("../findAllIframes");
       </html>`, {
       waitUntil: "domcontentloaded"
     });
-    const iframeNodes = await (0, _findAllIframes.findAllIframes)(page);
+    const iframeTree = await _frameTree.FrameTree.fromRoot(page);
+    const iframeNodes = Array.from(iframeTree.tree());
     (0, _extendedTest.expect)(iframeNodes.length).toBe(2);
     let sandboxedNode;
     let normalNode;