@taodigital/crawler 0.0.0-stage → 1.2.0

package/src/multi-context-handler.ts

@@ -0,0 +1,296 @@
+/**
+ * Multi-context browser handling — detects and records new tabs, iframes,
+ * popups, and file downloads triggered during a page crawl.
+ * This lets ZeTa discover cross-origin flows, OAuth redirects, embedded
+ * third-party content, and downloadable resources that single-page HTTP
+ * crawlers would miss entirely.
+ */
+
+import type { Page, BrowserContext } from 'playwright';
+
+export interface NewTabRecord {
+  url: string;
+  linkText: string;
+  isInternal: boolean;
+  httpStatus?: number;
+}
+
+export interface IframeRecord {
+  src: string;
+  crossOrigin: boolean;
+  sandboxed: boolean;
+  sandboxValue: string | null;
+  accessible: boolean;
+  contentSample: string;
+  hasForm: boolean;
+}
+
+export interface PopupRecord {
+  url: string;
+  triggerLabel: string;
+  isOAuthLike: boolean;
+}
+
+export interface DownloadRecord {
+  filename: string;
+  downloadUrl: string;
+  triggerLabel: string;
+}
+
+export interface MultiContextResult {
+  newTabs: NewTabRecord[];
+  iframes: IframeRecord[];
+  popups: PopupRecord[];
+  downloads: DownloadRecord[];
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+async function enumerateIframes(
+  page: Page,
+  seedUrl: string,
+): Promise<IframeRecord[]> {
+  const records: IframeRecord[] = [];
+  try {
+    const seedHost = new URL(seedUrl).hostname;
+    const iframeLocators = await page.locator('iframe').all();
+    const limited = iframeLocators.slice(0, 20);
+
+    for (let idx = 0; idx < limited.length; idx++) {
+      try {
+        const loc = limited[idx];
+        const src = (await loc.getAttribute('src')) ?? '';
+        const sandboxValue = await loc.getAttribute('sandbox');
+        const sandboxed = sandboxValue !== null;
+
+        let crossOrigin = false;
+        if (src) {
+          try {
+            const iframeHost = new URL(src, seedUrl).hostname;
+            crossOrigin = iframeHost !== seedHost;
+          } catch {
+            crossOrigin = false;
+          }
+        }
+
+        let accessible = false;
+        let contentSample = '';
+        let hasForm = false;
+
+        try {
+          const fl = page.frameLocator('iframe').nth(idx);
+          const body = await fl.locator('body').textContent({ timeout: 2000 });
+          if (body !== null) {
+            accessible = true;
+            contentSample = body.trim().slice(0, 300);
+          }
+          const formCount = await fl.locator('form, input, textarea, select').count();
+          hasForm = formCount > 0;
+        } catch {
+          // cross-origin or sandboxed — not accessible from parent frame
+        }
+
+        records.push({
+          src,
+          crossOrigin,
+          sandboxed,
+          sandboxValue,
+          accessible,
+          contentSample,
+          hasForm,
+        });
+      } catch {
+        // skip individual iframe errors
+      }
+    }
+  } catch {
+    // skip entirely if locator fails
+  }
+  return records;
+}
+
+async function detectPopups(page: Page, result: MultiContextResult): Promise<void> {
+  try {
+    const candidates = [
+      ...await page.locator('[onclick*="window.open"]').all(),
+      ...await page.locator('button:has-text("Open in")').all(),
+    ];
+    const limited = candidates.slice(0, 3);
+
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? '';
+        const outcome = await Promise.all([
+          page.waitForEvent('popup', { timeout: 2000 }),
+          trigger.click(),
+        ]).catch(() => null);
+
+        if (!outcome) continue;
+        const popup = outcome[0] as Page;
+        if (!popup) continue;
+
+        const url = popup.url();
+        let isOAuthLike = false;
+        try {
+          const host = new URL(url).hostname;
+          isOAuthLike = /google|github|microsoft|okta|auth0/.test(host);
+        } catch {
+          // not a valid URL
+        }
+
+        result.popups.push({ url, triggerLabel, isOAuthLike });
+        await popup.close().catch(() => {});
+      } catch {
+        // skip this trigger
+      }
+    }
+  } catch {
+    // skip popup detection entirely
+  }
+}
+
+async function detectDownloads(page: Page, result: MultiContextResult): Promise<void> {
+  try {
+    const candidates = [
+      ...await page.locator('a[download]').all(),
+      ...await page.locator('button:has-text("Export")').all(),
+      ...await page.locator('button:has-text("Download")').all(),
+    ];
+    const limited = candidates.slice(0, 3);
+
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? '';
+        const outcome = await Promise.all([
+          page.waitForEvent('download', { timeout: 2000 }),
+          trigger.click(),
+        ]).catch(() => null);
+
+        if (!outcome) continue;
+        const download = outcome[0] as unknown as import('playwright').Download;
+        if (!download) continue;
+
+        const filename = download.suggestedFilename();
+        const downloadUrl = download.url();
+        await download.cancel().catch(() => {});
+
+        result.downloads.push({ filename, downloadUrl, triggerLabel });
+      } catch {
+        // skip this trigger
+      }
+    }
+  } catch {
+    // skip download detection entirely
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exported functions
+// ---------------------------------------------------------------------------
+
+/**
+ * Attaches a listener to the BrowserContext that records every new tab
+ * opened while the listener is active. Returns an unsubscribe function.
+ *
+ * Call this before navigating / clicking so that tabs opened during the
+ * crawl are captured, then call the returned function to stop listening.
+ */
+export function attachNewTabListener(
+  context: BrowserContext,
+  seedUrl: string,
+  result: MultiContextResult,
+): () => void {
+  let seedHost = '';
+  try {
+    seedHost = new URL(seedUrl).hostname;
+  } catch {
+    // leave empty — all tabs will be treated as external
+  }
+
+  const handler = async (newPage: Page) => {
+    try {
+      await newPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});
+      const url = newPage.url();
+
+      if (!url || url === 'about:blank') {
+        await newPage.close().catch(() => {});
+        return;
+      }
+
+      let isInternal = false;
+      let httpStatus: number | undefined;
+
+      try {
+        const tabHost = new URL(url).hostname;
+        isInternal = tabHost === seedHost;
+      } catch {
+        // non-parseable URL — treat as external
+      }
+
+      if (!isInternal) {
+        try {
+          const controller = new AbortController();
+          const timer = setTimeout(() => controller.abort(), 3000);
+          const resp = await fetch(url, { method: 'HEAD', signal: controller.signal });
+          clearTimeout(timer);
+          httpStatus = resp.status;
+        } catch {
+          // network unreachable or CORS — leave httpStatus undefined
+        }
+      }
+
+      // linkText is not available at context level; caller fills it if needed
+      result.newTabs.push({ url, linkText: '', isInternal, httpStatus });
+      await newPage.close().catch(() => {});
+    } catch {
+      // swallow handler errors so the listener stays alive
+    }
+  };
+
+  context.on('page', handler);
+  return () => context.off('page', handler);
+}
+
+/**
+ * Probes the current page for multi-context browser events: iframes,
+ * popup windows, and file downloads.
+ *
+ * New-tab tracking is intentionally excluded here — it must be set up at
+ * the crawler level (before navigation) using `attachNewTabListener`.
+ */
+export async function probeMultiContext(
+  page: Page,
+  context: BrowserContext,
+  seedUrl: string,
+): Promise<MultiContextResult> {
+  void context; // reserved for future use (e.g. routing overrides)
+
+  const result: MultiContextResult = {
+    newTabs: [],
+    iframes: [],
+    popups: [],
+    downloads: [],
+  };
+
+  try {
+    result.iframes = await enumerateIframes(page, seedUrl);
+  } catch {
+    // keep empty
+  }
+
+  try {
+    await detectPopups(page, result);
+  } catch {
+    // keep empty
+  }
+
+  try {
+    await detectDownloads(page, result);
+  } catch {
+    // keep empty
+  }
+
+  return result;
+}