@ddwang/magnitude-core 0.3.1-ddwang.3 → 0.3.1-ddwang.4

package/dist/agent/browserAgent.js

@@ -6,6 +6,7 @@ import { partitionHtml, serializeToMarkdown } from 'magnitude-extract';
 import EventEmitter from "eventemitter3";
 import { retry } from "@/common/retry";
 import { checkOperation } from '@/common/operation';
+import { getVisiblePageContent } from '@/web/pageContent';
 // export interface StartAgentWithWebOptions {
 //     agentBaseOptions?: Partial<AgentOptions>;
 //     webConnectorOptions?: BrowserConnectorOptions;
@@ -29,47 +30,6 @@ export async function startBrowserAgent(options //StartAgentWithWebOptions = {}
     //console.log('agent started');
     return agent;
 }
-async function getFullPageContent(page) {
-    checkOperation();
-    // 1. Get all iframe element handles
-    const iframeHandles = await page.locator('iframe').elementHandles();
-    // 2. Iterate through each iframe handle
-    for (const iframeHandle of iframeHandles) {
-        checkOperation();
-        // 3. Get the Frame object for the iframe
-        const frame = await iframeHandle.contentFrame();
-        checkOperation();
-        if (frame) {
-            // 4. Get the HTML content of the iframe
-            const iframeContent = await frame.content();
-            checkOperation();
-            // 5. Use evaluate to replace the iframe element with its content.
-            // We pass the content as an argument to avoid issues with string escaping.
-            await iframeHandle.evaluate((iframeNode, { content }) => {
-                // Create a new div element to hold the iframe's content
-                const div = document.createElement('div');
-                // Use DOMParser to handle Trusted Types restrictions
-                const parser = new DOMParser();
-                const doc = parser.parseFromString(content, 'text/html');
-                // Move all body children to the div
-                while (doc.body.firstChild) {
-                    div.appendChild(doc.body.firstChild);
-                }
-                // Also preserve any head elements that might be important (styles, etc)
-                const headElements = doc.head.querySelectorAll('style, link[rel="stylesheet"]');
-                headElements.forEach(el => div.appendChild(el.cloneNode(true)));
-                // Add a data-attribute to mark that this was an expanded iframe
-                div.dataset.expandedFromIframe = 'true';
-                div.dataset.iframeSrc = iframeNode.getAttribute('src') || '';
-                // Replace the iframeNode with the new div
-                iframeNode.parentNode?.replaceChild(div, iframeNode);
-            }, { content: iframeContent });
-        }
-    }
-    // 6. Return the final, modified page content
-    checkOperation();
-    return page.content();
-}
 export class BrowserAgent extends Agent {
     browserAgentEvents = new EventEmitter();
     constructor({ agentOptions, browserOptions }) {
@@ -98,8 +58,7 @@ export class BrowserAgent extends Agent {
     }
     async _extract(instructions, schema) {
         this.browserAgentEvents.emit('extractStarted', instructions, schema);
-        //const htmlContent = await this.page.content();
-        const htmlContent = await retry(async () => await getFullPageContent(this.page), { retries: 5, delay: 200, exponential: true });
+        const htmlContent = await retry(() => getVisiblePageContent(this.page), { retries: 5, delay: 200, exponential: true });
         // const accessibilityTree = await this.page.accessibility.snapshot({ interestingOnly: true });
         // const pageRepr = renderMinimalAccessibilityTree(accessibilityTree);
         const partitionOptions = {

package/dist/index.cjs

@@ -7200,6 +7200,72 @@ function narrateBrowserAgent(agent) {
   });
 }
 
+async function getVisiblePageContent(page) {
+  return readFrame(page.mainFrame());
+}
+async function readFrame(frame) {
+  checkOperation();
+  const marker = crypto.randomUUID();
+  const snapshot = await frame.evaluateHandle((marker2) => {
+    const documentCopy = document.implementation.createHTMLDocument();
+    const frames = [];
+    const visibility = { checkOpacity: true, checkVisibility: true };
+    function copy(source) {
+      if (["script", "style", "template", "noscript"].includes(source.localName)) return null;
+      const style = getComputedStyle(source);
+      if (style.display === "none" || style.opacity === "0" || style.contentVisibility === "hidden") return null;
+      const visible = style.visibility === "visible" && (style.display === "contents" || source.checkVisibility(visibility) || ["option", "optgroup"].includes(source.localName) && source.closest("select")?.checkVisibility(visibility));
+      if (source instanceof HTMLIFrameElement || source instanceof HTMLFrameElement) {
+        if (!visible) return null;
+        frames.push(source);
+        return documentCopy.createComment(`${marker2}:${frames.length - 1}`);
+      }
+      const target = documentCopy.importNode(source, false);
+      if (source instanceof HTMLInputElement) {
+        if (source.type === "password") target.removeAttribute("value");
+        else target.setAttribute("value", source.value);
+        target.toggleAttribute("checked", source.checked);
+      }
+      if (source instanceof HTMLOptionElement) target.toggleAttribute("selected", source.selected);
+      if (source instanceof HTMLTextAreaElement) {
+        target.textContent = visible ? source.value : "";
+        return visible ? target : null;
+      }
+      const summary = source instanceof HTMLDetailsElement && !source.open ? source.querySelector("summary") : null;
+      for (const child of source.childNodes) {
+        if (source instanceof HTMLDetailsElement && !source.open && child !== summary) continue;
+        if (child instanceof Element) {
+          const copied = copy(child);
+          if (copied) target.appendChild(copied);
+        } else if (visible && child.nodeType === Node.TEXT_NODE) {
+          target.appendChild(documentCopy.importNode(child, false));
+        }
+      }
+      return visible || target.hasChildNodes() ? target : null;
+    }
+    const body = document.body && copy(document.body);
+    return { html: body instanceof Element ? body.innerHTML : "", frames };
+  }, marker);
+  const handles = [snapshot];
+  try {
+    let html = await snapshot.evaluate(({ html: html2 }) => html2);
+    const frames = await snapshot.getProperty("frames");
+    handles.push(frames);
+    const entries = await frames.getProperties();
+    handles.push(...entries.values());
+    for (const [index, handle] of entries) {
+      checkOperation();
+      const child = await handle.asElement()?.contentFrame();
+      const content = child ? await readFrame(child) : "";
+      html = html.replace(`<!--${marker}:${index}-->`, () => `<div>${content}</div>`);
+    }
+    checkOperation();
+    return html;
+  } finally {
+    await Promise.all(handles.map((handle) => handle.dispose()));
+  }
+}
+
 async function startBrowserAgent(options) {
   const { agentOptions, browserOptions } = buildDefaultBrowserAgentOptions({ agentOptions: options ?? {}, browserOptions: options ?? {} });
   const agent = new BrowserAgent({
@@ -7212,34 +7278,6 @@ async function startBrowserAgent(options) {
   await agent.start();
   return agent;
 }
-async function getFullPageContent(page) {
-  checkOperation();
-  const iframeHandles = await page.locator("iframe").elementHandles();
-  for (const iframeHandle of iframeHandles) {
-    checkOperation();
-    const frame = await iframeHandle.contentFrame();
-    checkOperation();
-    if (frame) {
-      const iframeContent = await frame.content();
-      checkOperation();
-      await iframeHandle.evaluate((iframeNode, { content }) => {
-        const div = document.createElement("div");
-        const parser = new DOMParser();
-        const doc = parser.parseFromString(content, "text/html");
-        while (doc.body.firstChild) {
-          div.appendChild(doc.body.firstChild);
-        }
-        const headElements = doc.head.querySelectorAll('style, link[rel="stylesheet"]');
-        headElements.forEach((el) => div.appendChild(el.cloneNode(true)));
-        div.dataset.expandedFromIframe = "true";
-        div.dataset.iframeSrc = iframeNode.getAttribute("src") || "";
-        iframeNode.parentNode?.replaceChild(div, iframeNode);
-      }, { content: iframeContent });
-    }
-  }
-  checkOperation();
-  return page.content();
-}
 class BrowserAgent extends Agent {
   browserAgentEvents = new EventEmitter();
   constructor({ agentOptions, browserOptions }) {
@@ -7267,7 +7305,7 @@ class BrowserAgent extends Agent {
   async _extract(instructions, schema) {
     this.browserAgentEvents.emit("extractStarted", instructions, schema);
     const htmlContent = await retry(
-      async () => await getFullPageContent(this.page),
+      () => getVisiblePageContent(this.page),
       { retries: 5, delay: 200, exponential: true }
     );
     const partitionOptions = {

package/dist/index.mjs

@@ -7177,6 +7177,72 @@ function narrateBrowserAgent(agent) {
   });
 }
 
+async function getVisiblePageContent(page) {
+  return readFrame(page.mainFrame());
+}
+async function readFrame(frame) {
+  checkOperation();
+  const marker = randomUUID();
+  const snapshot = await frame.evaluateHandle((marker2) => {
+    const documentCopy = document.implementation.createHTMLDocument();
+    const frames = [];
+    const visibility = { checkOpacity: true, checkVisibility: true };
+    function copy(source) {
+      if (["script", "style", "template", "noscript"].includes(source.localName)) return null;
+      const style = getComputedStyle(source);
+      if (style.display === "none" || style.opacity === "0" || style.contentVisibility === "hidden") return null;
+      const visible = style.visibility === "visible" && (style.display === "contents" || source.checkVisibility(visibility) || ["option", "optgroup"].includes(source.localName) && source.closest("select")?.checkVisibility(visibility));
+      if (source instanceof HTMLIFrameElement || source instanceof HTMLFrameElement) {
+        if (!visible) return null;
+        frames.push(source);
+        return documentCopy.createComment(`${marker2}:${frames.length - 1}`);
+      }
+      const target = documentCopy.importNode(source, false);
+      if (source instanceof HTMLInputElement) {
+        if (source.type === "password") target.removeAttribute("value");
+        else target.setAttribute("value", source.value);
+        target.toggleAttribute("checked", source.checked);
+      }
+      if (source instanceof HTMLOptionElement) target.toggleAttribute("selected", source.selected);
+      if (source instanceof HTMLTextAreaElement) {
+        target.textContent = visible ? source.value : "";
+        return visible ? target : null;
+      }
+      const summary = source instanceof HTMLDetailsElement && !source.open ? source.querySelector("summary") : null;
+      for (const child of source.childNodes) {
+        if (source instanceof HTMLDetailsElement && !source.open && child !== summary) continue;
+        if (child instanceof Element) {
+          const copied = copy(child);
+          if (copied) target.appendChild(copied);
+        } else if (visible && child.nodeType === Node.TEXT_NODE) {
+          target.appendChild(documentCopy.importNode(child, false));
+        }
+      }
+      return visible || target.hasChildNodes() ? target : null;
+    }
+    const body = document.body && copy(document.body);
+    return { html: body instanceof Element ? body.innerHTML : "", frames };
+  }, marker);
+  const handles = [snapshot];
+  try {
+    let html = await snapshot.evaluate(({ html: html2 }) => html2);
+    const frames = await snapshot.getProperty("frames");
+    handles.push(frames);
+    const entries = await frames.getProperties();
+    handles.push(...entries.values());
+    for (const [index, handle] of entries) {
+      checkOperation();
+      const child = await handle.asElement()?.contentFrame();
+      const content = child ? await readFrame(child) : "";
+      html = html.replace(`<!--${marker}:${index}-->`, () => `<div>${content}</div>`);
+    }
+    checkOperation();
+    return html;
+  } finally {
+    await Promise.all(handles.map((handle) => handle.dispose()));
+  }
+}
+
 async function startBrowserAgent(options) {
   const { agentOptions, browserOptions } = buildDefaultBrowserAgentOptions({ agentOptions: options ?? {}, browserOptions: options ?? {} });
   const agent = new BrowserAgent({
@@ -7189,34 +7255,6 @@ async function startBrowserAgent(options) {
   await agent.start();
   return agent;
 }
-async function getFullPageContent(page) {
-  checkOperation();
-  const iframeHandles = await page.locator("iframe").elementHandles();
-  for (const iframeHandle of iframeHandles) {
-    checkOperation();
-    const frame = await iframeHandle.contentFrame();
-    checkOperation();
-    if (frame) {
-      const iframeContent = await frame.content();
-      checkOperation();
-      await iframeHandle.evaluate((iframeNode, { content }) => {
-        const div = document.createElement("div");
-        const parser = new DOMParser();
-        const doc = parser.parseFromString(content, "text/html");
-        while (doc.body.firstChild) {
-          div.appendChild(doc.body.firstChild);
-        }
-        const headElements = doc.head.querySelectorAll('style, link[rel="stylesheet"]');
-        headElements.forEach((el) => div.appendChild(el.cloneNode(true)));
-        div.dataset.expandedFromIframe = "true";
-        div.dataset.iframeSrc = iframeNode.getAttribute("src") || "";
-        iframeNode.parentNode?.replaceChild(div, iframeNode);
-      }, { content: iframeContent });
-    }
-  }
-  checkOperation();
-  return page.content();
-}
 class BrowserAgent extends Agent {
   browserAgentEvents = new EventEmitter();
   constructor({ agentOptions, browserOptions }) {
@@ -7244,7 +7282,7 @@ class BrowserAgent extends Agent {
   async _extract(instructions, schema) {
     this.browserAgentEvents.emit("extractStarted", instructions, schema);
     const htmlContent = await retry(
-      async () => await getFullPageContent(this.page),
+      () => getVisiblePageContent(this.page),
       { retries: 5, delay: 200, exponential: true }
     );
     const partitionOptions = {

package/dist/web/pageContent.d.ts

@@ -0,0 +1,2 @@
+import type { Page } from 'playwright';
+export declare function getVisiblePageContent(page: Page): Promise<string>;

package/dist/web/pageContent.js

@@ -0,0 +1,78 @@
+import { randomUUID } from 'node:crypto';
+import { checkOperation } from '@/common/operation';
+export async function getVisiblePageContent(page) {
+    return readFrame(page.mainFrame());
+}
+async function readFrame(frame) {
+    checkOperation();
+    const marker = randomUUID();
+    const snapshot = await frame.evaluateHandle((marker) => {
+        const documentCopy = document.implementation.createHTMLDocument();
+        const frames = [];
+        const visibility = { checkOpacity: true, checkVisibility: true };
+        function copy(source) {
+            if (['script', 'style', 'template', 'noscript'].includes(source.localName))
+                return null;
+            const style = getComputedStyle(source);
+            if (style.display === 'none' || style.opacity === '0' || style.contentVisibility === 'hidden')
+                return null;
+            const visible = style.visibility === 'visible' && (style.display === 'contents' || source.checkVisibility(visibility) ||
+                (['option', 'optgroup'].includes(source.localName) && source.closest('select')?.checkVisibility(visibility)));
+            if (source instanceof HTMLIFrameElement || source instanceof HTMLFrameElement) {
+                if (!visible)
+                    return null;
+                frames.push(source);
+                return documentCopy.createComment(`${marker}:${frames.length - 1}`);
+            }
+            const target = documentCopy.importNode(source, false);
+            if (source instanceof HTMLInputElement) {
+                if (source.type === 'password')
+                    target.removeAttribute('value');
+                else
+                    target.setAttribute('value', source.value);
+                target.toggleAttribute('checked', source.checked);
+            }
+            if (source instanceof HTMLOptionElement)
+                target.toggleAttribute('selected', source.selected);
+            if (source instanceof HTMLTextAreaElement) {
+                target.textContent = visible ? source.value : '';
+                return visible ? target : null;
+            }
+            const summary = source instanceof HTMLDetailsElement && !source.open ? source.querySelector('summary') : null;
+            for (const child of source.childNodes) {
+                if (source instanceof HTMLDetailsElement && !source.open && child !== summary)
+                    continue;
+                if (child instanceof Element) {
+                    const copied = copy(child);
+                    if (copied)
+                        target.appendChild(copied);
+                }
+                else if (visible && child.nodeType === Node.TEXT_NODE) {
+                    target.appendChild(documentCopy.importNode(child, false));
+                }
+            }
+            return visible || target.hasChildNodes() ? target : null;
+        }
+        const body = document.body && copy(document.body);
+        return { html: body instanceof Element ? body.innerHTML : '', frames };
+    }, marker);
+    const handles = [snapshot];
+    try {
+        let html = await snapshot.evaluate(({ html }) => html);
+        const frames = await snapshot.getProperty('frames');
+        handles.push(frames);
+        const entries = await frames.getProperties();
+        handles.push(...entries.values());
+        for (const [index, handle] of entries) {
+            checkOperation();
+            const child = await handle.asElement()?.contentFrame();
+            const content = child ? await readFrame(child) : '';
+            html = html.replace(`<!--${marker}:${index}-->`, () => `<div>${content}</div>`);
+        }
+        checkOperation();
+        return html;
+    }
+    finally {
+        await Promise.all(handles.map(handle => handle.dispose()));
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ddwang/magnitude-core",
-  "version": "0.3.1-ddwang.3",
+  "version": "0.3.1-ddwang.4",
   "description": "Magnitude e2e testing agent",
   "publishConfig": {
     "access": "public"