@detiq/crawler 1.1.14 → 1.1.16

package/src/locator-extractor.ts

@@ -220,9 +220,35 @@ export function elementToPlaywrightLocator(element: {
   return null;
 }
 
+// Patterns that indicate crawl-analysis meta-text stored as element records (not real UI elements).
+// These come from accessibility-auditor, screen-reader analysis, or CSS inspector annotations.
+const META_ANNOTATION_RE = /^(css animations|scroll effects detected|design tokens?\s*:|font imports?:|contrast ratio|aria-label audit|landmark|wcag|color contrast|background.*foreground.*card|--[\w-]+,\s*--)/i;
+
+// Auth-wall element patterns — if >50% of locators match these, the page was an auth wall.
+const AUTH_WALL_TEXT_RE = /sign\s*in|log\s*in|platform access required|please sign in|access required|authentication required|session expired|unauthorized/i;
+
+/**
+ * Returns true if an element's text content is crawl-analysis meta-text, not a real UI element.
+ * These are annotations stored by the accessibility auditor, not actual DOM elements.
+ */
+function isMetaAnnotation(el: { role?: string | null; meaning?: string | null; text?: string | null }): boolean {
+  const t = (el.meaning ?? el.text ?? '').trim();
+  if (!t) return false;
+  // Filter crawl-analysis annotations
+  if (META_ANNOTATION_RE.test(t)) return true;
+  // Filter very long text strings with no interactive role — these are paragraph/body copy,
+  // not useful UI locators. Threshold: 120 chars with no testId/label/role
+  const interactiveRoles = new Set(['button', 'link', 'textbox', 'combobox', 'checkbox', 'radio', 'tab', 'option', 'menuitem']);
+  if (t.length > 120 && !interactiveRoles.has((el.role ?? '').toLowerCase())) return true;
+  // Filter CSS variable dumps (contain multiple --var-name patterns)
+  if ((t.match(/--[\w-]+/g) ?? []).length > 3) return true;
+  return false;
+}
+
 /**
  * Convert an array of DB Element records to deduplicated Playwright locators.
  * Deduplicates by key — higher confidence wins.
+ * Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.
  */
 export function elementsToPlaywrightLocators(elements: Array<{
   role?: string | null;
@@ -237,6 +263,9 @@ export function elementsToPlaywrightLocators(elements: Array<{
   const byKey = new Map<string, PlaywrightLocator>();
 
   for (const el of elements) {
+    // Skip crawl-analysis meta-annotations — these are not real page elements
+    if (isMetaAnnotation(el)) continue;
+
     const loc = elementToPlaywrightLocator(el);
     if (!loc) continue;
     const existing = byKey.get(loc.key);
@@ -248,6 +277,22 @@ export function elementsToPlaywrightLocators(elements: Array<{
   return Array.from(byKey.values());
 }
 
+/**
+ * Detect whether a set of locators was extracted from an auth-wall page.
+ * Returns a warning message if auth-wall detected, null otherwise.
+ * Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.
+ */
+export function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null {
+  if (locators.length === 0) return null;
+  const lowCount = locators.filter(l => l.confidence === 'low').length;
+  const hasAuthText = locators.some(l => AUTH_WALL_TEXT_RE.test(l.description));
+  const highCount = locators.filter(l => l.confidence === 'high').length;
+  if (hasAuthText && highCount === 0 && lowCount / locators.length > 0.5) {
+    return 'These locators appear to be from a login/access wall, not the target page. The page requires authentication. Add credentials in Project Settings and re-crawl to get accurate locators for the protected content.';
+  }
+  return null;
+}
+
 /**
  * Format locators as a block for injection into LLM script generation prompts.
  * Returns empty string if no locators.

package/src/multi-context-handler.ts

@@ -0,0 +1,296 @@
+/**
+ * Multi-context browser handling — detects and records new tabs, iframes,
+ * popups, and file downloads triggered during a page crawl.
+ * This lets ZeTa discover cross-origin flows, OAuth redirects, embedded
+ * third-party content, and downloadable resources that single-page HTTP
+ * crawlers would miss entirely.
+ */
+
+import type { Page, BrowserContext } from 'playwright';
+
+export interface NewTabRecord {
+  url: string;
+  linkText: string;
+  isInternal: boolean;
+  httpStatus?: number;
+}
+
+export interface IframeRecord {
+  src: string;
+  crossOrigin: boolean;
+  sandboxed: boolean;
+  sandboxValue: string | null;
+  accessible: boolean;
+  contentSample: string;
+  hasForm: boolean;
+}
+
+export interface PopupRecord {
+  url: string;
+  triggerLabel: string;
+  isOAuthLike: boolean;
+}
+
+export interface DownloadRecord {
+  filename: string;
+  downloadUrl: string;
+  triggerLabel: string;
+}
+
+export interface MultiContextResult {
+  newTabs: NewTabRecord[];
+  iframes: IframeRecord[];
+  popups: PopupRecord[];
+  downloads: DownloadRecord[];
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+async function enumerateIframes(
+  page: Page,
+  seedUrl: string,
+): Promise<IframeRecord[]> {
+  const records: IframeRecord[] = [];
+  try {
+    const seedHost = new URL(seedUrl).hostname;
+    const iframeLocators = await page.locator('iframe').all();
+    const limited = iframeLocators.slice(0, 20);
+
+    for (let idx = 0; idx < limited.length; idx++) {
+      try {
+        const loc = limited[idx];
+        const src = (await loc.getAttribute('src')) ?? '';
+        const sandboxValue = await loc.getAttribute('sandbox');
+        const sandboxed = sandboxValue !== null;
+
+        let crossOrigin = false;
+        if (src) {
+          try {
+            const iframeHost = new URL(src, seedUrl).hostname;
+            crossOrigin = iframeHost !== seedHost;
+          } catch {
+            crossOrigin = false;
+          }
+        }
+
+        let accessible = false;
+        let contentSample = '';
+        let hasForm = false;
+
+        try {
+          const fl = page.frameLocator('iframe').nth(idx);
+          const body = await fl.locator('body').textContent({ timeout: 2000 });
+          if (body !== null) {
+            accessible = true;
+            contentSample = body.trim().slice(0, 300);
+          }
+          const formCount = await fl.locator('form, input, textarea, select').count();
+          hasForm = formCount > 0;
+        } catch {
+          // cross-origin or sandboxed — not accessible from parent frame
+        }
+
+        records.push({
+          src,
+          crossOrigin,
+          sandboxed,
+          sandboxValue,
+          accessible,
+          contentSample,
+          hasForm,
+        });
+      } catch {
+        // skip individual iframe errors
+      }
+    }
+  } catch {
+    // skip entirely if locator fails
+  }
+  return records;
+}
+
+async function detectPopups(page: Page, result: MultiContextResult): Promise<void> {
+  try {
+    const candidates = [
+      ...await page.locator('[onclick*="window.open"]').all(),
+      ...await page.locator('button:has-text("Open in")').all(),
+    ];
+    const limited = candidates.slice(0, 3);
+
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? '';
+        const outcome = await Promise.all([
+          page.waitForEvent('popup', { timeout: 2000 }),
+          trigger.click(),
+        ]).catch(() => null);
+
+        if (!outcome) continue;
+        const popup = outcome[0] as Page;
+        if (!popup) continue;
+
+        const url = popup.url();
+        let isOAuthLike = false;
+        try {
+          const host = new URL(url).hostname;
+          isOAuthLike = /google|github|microsoft|okta|auth0/.test(host);
+        } catch {
+          // not a valid URL
+        }
+
+        result.popups.push({ url, triggerLabel, isOAuthLike });
+        await popup.close().catch(() => {});
+      } catch {
+        // skip this trigger
+      }
+    }
+  } catch {
+    // skip popup detection entirely
+  }
+}
+
+async function detectDownloads(page: Page, result: MultiContextResult): Promise<void> {
+  try {
+    const candidates = [
+      ...await page.locator('a[download]').all(),
+      ...await page.locator('button:has-text("Export")').all(),
+      ...await page.locator('button:has-text("Download")').all(),
+    ];
+    const limited = candidates.slice(0, 3);
+
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? '';
+        const outcome = await Promise.all([
+          page.waitForEvent('download', { timeout: 2000 }),
+          trigger.click(),
+        ]).catch(() => null);
+
+        if (!outcome) continue;
+        const download = outcome[0] as unknown as import('playwright').Download;
+        if (!download) continue;
+
+        const filename = download.suggestedFilename();
+        const downloadUrl = download.url();
+        await download.cancel().catch(() => {});
+
+        result.downloads.push({ filename, downloadUrl, triggerLabel });
+      } catch {
+        // skip this trigger
+      }
+    }
+  } catch {
+    // skip download detection entirely
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exported functions
+// ---------------------------------------------------------------------------
+
+/**
+ * Attaches a listener to the BrowserContext that records every new tab
+ * opened while the listener is active. Returns an unsubscribe function.
+ *
+ * Call this before navigating / clicking so that tabs opened during the
+ * crawl are captured, then call the returned function to stop listening.
+ */
+export function attachNewTabListener(
+  context: BrowserContext,
+  seedUrl: string,
+  result: MultiContextResult,
+): () => void {
+  let seedHost = '';
+  try {
+    seedHost = new URL(seedUrl).hostname;
+  } catch {
+    // leave empty — all tabs will be treated as external
+  }
+
+  const handler = async (newPage: Page) => {
+    try {
+      await newPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});
+      const url = newPage.url();
+
+      if (!url || url === 'about:blank') {
+        await newPage.close().catch(() => {});
+        return;
+      }
+
+      let isInternal = false;
+      let httpStatus: number | undefined;
+
+      try {
+        const tabHost = new URL(url).hostname;
+        isInternal = tabHost === seedHost;
+      } catch {
+        // non-parseable URL — treat as external
+      }
+
+      if (!isInternal) {
+        try {
+          const controller = new AbortController();
+          const timer = setTimeout(() => controller.abort(), 3000);
+          const resp = await fetch(url, { method: 'HEAD', signal: controller.signal });
+          clearTimeout(timer);
+          httpStatus = resp.status;
+        } catch {
+          // network unreachable or CORS — leave httpStatus undefined
+        }
+      }
+
+      // linkText is not available at context level; caller fills it if needed
+      result.newTabs.push({ url, linkText: '', isInternal, httpStatus });
+      await newPage.close().catch(() => {});
+    } catch {
+      // swallow handler errors so the listener stays alive
+    }
+  };
+
+  context.on('page', handler);
+  return () => context.off('page', handler);
+}
+
+/**
+ * Probes the current page for multi-context browser events: iframes,
+ * popup windows, and file downloads.
+ *
+ * New-tab tracking is intentionally excluded here — it must be set up at
+ * the crawler level (before navigation) using `attachNewTabListener`.
+ */
+export async function probeMultiContext(
+  page: Page,
+  context: BrowserContext,
+  seedUrl: string,
+): Promise<MultiContextResult> {
+  void context; // reserved for future use (e.g. routing overrides)
+
+  const result: MultiContextResult = {
+    newTabs: [],
+    iframes: [],
+    popups: [],
+    downloads: [],
+  };
+
+  try {
+    result.iframes = await enumerateIframes(page, seedUrl);
+  } catch {
+    // keep empty
+  }
+
+  try {
+    await detectPopups(page, result);
+  } catch {
+    // keep empty
+  }
+
+  try {
+    await detectDownloads(page, result);
+  } catch {
+    // keep empty
+  }
+
+  return result;
+}

package/src/nested-window-crawler.ts

@@ -0,0 +1,310 @@
+/**
+ * Nested window / browsing-context crawler — discovers and records frames,
+ * embedded apps, and popup windows that are already open in the browser
+ * context at the time of a crawl step.
+ *
+ * Complements multi-context-handler.ts: that module handles proactive popup
+ * interception via context.on('page'); this module snapshots what is already
+ * present in the frame tree and context page list.
+ */
+
+import type { Page, BrowserContext } from 'playwright';
+
+export type BrowsingContextType = 'page' | 'frame' | 'popup' | 'embedded-app';
+
+export interface BrowsingContextNode {
+  id: string;
+  type: BrowsingContextType;
+  url: string;
+  parentId: string | null;
+  depth: number;
+  title: string;
+  interactiveCount: number;
+  accessible: boolean;
+  isEmbeddedApp: boolean;
+}
+
+export interface NestedCrawlResult {
+  contextTree: BrowsingContextNode[];
+  totalContextsFound: number;
+  maxDepthReached: number;
+  inaccessibleCount: number;
+  frameUrls: string[];
+  embeddedAppUrls: string[];
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Deterministic 12-char hex id derived from context type + URL via FNV-1a.
+ * No external deps — algorithm implemented inline.
+ */
+function generateContextId(type: string, url: string): string {
+  let h = 0x811c9dc5;
+  const s = type + url;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = (h * 0x01000193) >>> 0;
+  }
+  return h.toString(16).padStart(8, '0').slice(0, 12);
+}
+
+/**
+ * Returns the frame depth by walking the parentFrame() chain.
+ * The main frame has depth 0.
+ */
+function computeFrameDepth(frame: import('playwright').Frame): number {
+  let depth = 0;
+  let current = frame.parentFrame();
+  while (current !== null) {
+    depth++;
+    current = current.parentFrame();
+  }
+  return depth;
+}
+
+/**
+ * Walks all frames attached to the page and returns a BrowsingContextNode
+ * for each reachable, non-trivial frame up to maxDepth.
+ */
+async function crawlFrameTree(
+  page: any,
+  maxDepth = 4,
+): Promise<BrowsingContextNode[]> {
+  const nodes: BrowsingContextNode[] = [];
+
+  try {
+    const frames = page.frames();
+    const limited = frames.slice(0, 200);
+
+    for (const frame of limited) {
+      try {
+        const url = frame.url();
+
+        // Skip blank / synthetic frames
+        if (
+          !url ||
+          url === 'about:blank' ||
+          url.startsWith('javascript:') ||
+          url.startsWith('data:')
+        ) {
+          continue;
+        }
+
+        const depth = computeFrameDepth(frame);
+        if (depth > maxDepth) continue;
+
+        const parentFrame = frame.parentFrame();
+        const parentId = parentFrame
+          ? generateContextId('frame', parentFrame.url())
+          : null;
+
+        let title = '';
+        let interactiveCount = 0;
+        let accessible = false;
+
+        try {
+          title = await frame.title();
+          interactiveCount = await frame
+            .locator('button, a[href], input, select')
+            .count();
+          accessible = true;
+        } catch {
+          // cross-origin or sandboxed — treat as inaccessible
+        }
+
+        nodes.push({
+          id: generateContextId('frame', url),
+          type: 'frame',
+          url,
+          parentId,
+          depth,
+          title,
+          interactiveCount,
+          accessible,
+          isEmbeddedApp: false,
+        });
+      } catch {
+        // skip individual frame errors
+      }
+    }
+  } catch {
+    // skip entirely if frames() fails
+  }
+
+  return nodes;
+}
+
+/**
+ * Evaluates the page DOM to identify iframes that look like full embedded
+ * applications (large viewport coverage, or app-class/id/src signals).
+ * Returns up to 5 absolute URLs.
+ */
+async function detectEmbeddedApps(page: any): Promise<string[]> {
+  try {
+    const srcs = await (page.evaluate as any)(() => {
+      return Array.from(document.querySelectorAll('iframe'))
+        .filter((f) => {
+          const r = f.getBoundingClientRect();
+          const vw = window.innerWidth;
+          const vh = window.innerHeight;
+          const isLarge = r.width >= vw * 0.7 && r.height >= vh * 0.7;
+          const hasAppClass = /embed|dashboard|app-frame|portal/i.test(
+            f.className + f.id + (f.getAttribute('src') ?? ''),
+          );
+          return isLarge || hasAppClass;
+        })
+        .map((f) => f.src)
+        .filter(Boolean);
+    });
+
+    const baseUrl = page.url();
+    const resolved: string[] = [];
+
+    for (const src of srcs.slice(0, 5)) {
+      try {
+        resolved.push(new URL(src, baseUrl).href);
+      } catch {
+        // skip unparseable src
+      }
+    }
+
+    return resolved;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Snapshots all pages currently open in the context that are not the seed
+ * URL and records them as popup nodes at depth 1.
+ *
+ * Proactive interception (context.on('page')) is handled separately by
+ * attachNewTabListener in multi-context-handler.ts.
+ */
+async function crawlPopupChain(
+  context: BrowserContext,
+  seedUrl: string,
+): Promise<BrowsingContextNode[]> {
+  const nodes: BrowsingContextNode[] = [];
+
+  try {
+    const pages = context.pages();
+
+    for (const pg of pages) {
+      try {
+        const url = pg.url();
+
+        if (!url || url === 'about:blank' || url === seedUrl) continue;
+
+        let title = '';
+        let interactiveCount = 0;
+        let accessible = false;
+
+        try {
+          title = await pg.title();
+          interactiveCount = await pg
+            .locator('button, a[href], input, select')
+            .count();
+          accessible = true;
+        } catch {
+          // page may be closed or cross-origin restricted
+        }
+
+        nodes.push({
+          id: generateContextId('popup', url),
+          type: 'popup',
+          url,
+          parentId: null, // chain depth not reliably trackable for existing pages
+          depth: 1,
+          title,
+          interactiveCount,
+          accessible,
+          isEmbeddedApp: false,
+        });
+      } catch {
+        // skip individual page errors
+      }
+    }
+  } catch {
+    // skip if context.pages() fails
+  }
+
+  return nodes;
+}
+
+// ---------------------------------------------------------------------------
+// Exported entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Discovers all nested browsing contexts reachable from the current page:
+ * sub-frames, embedded applications, and already-open popup pages.
+ *
+ * All failures are non-fatal — errors are caught and an empty result is
+ * returned so the caller's crawl step can continue uninterrupted.
+ */
+export async function crawlNestedContexts(
+  page: any,
+  context: BrowserContext | null,
+  seedUrl: string,
+  opts?: { maxFrameDepth?: number; maxPopupDepth?: number },
+): Promise<NestedCrawlResult> {
+  const empty: NestedCrawlResult = {
+    contextTree: [],
+    totalContextsFound: 0,
+    maxDepthReached: 0,
+    inaccessibleCount: 0,
+    frameUrls: [],
+    embeddedAppUrls: [],
+  };
+
+  try {
+    const maxFrameDepth = opts?.maxFrameDepth ?? 4;
+
+    // Run independent discovery passes concurrently
+    const [frameNodes, embeddedAppUrls, popupNodes] = await Promise.all([
+      crawlFrameTree(page, maxFrameDepth).catch((): BrowsingContextNode[] => []),
+      detectEmbeddedApps(page).catch((): string[] => []),
+      context
+        ? crawlPopupChain(context, seedUrl).catch((): BrowsingContextNode[] => [])
+        : Promise.resolve<BrowsingContextNode[]>([]),
+    ]);
+
+    // Promote frame nodes that match an embedded-app URL
+    const embeddedSet = new Set(embeddedAppUrls);
+    for (const node of frameNodes) {
+      if (embeddedSet.has(node.url)) {
+        node.isEmbeddedApp = true;
+        node.type = 'embedded-app';
+      }
+    }
+
+    const contextTree: BrowsingContextNode[] = [...frameNodes, ...popupNodes];
+
+    // Deduplicate by id (same URL appearing in both lists)
+    const seen = new Set<string>();
+    const deduped = contextTree.filter((n) => {
+      if (seen.has(n.id)) return false;
+      seen.add(n.id);
+      return true;
+    });
+
+    const maxDepthReached = deduped.reduce((max, n) => Math.max(max, n.depth), 0);
+    const inaccessibleCount = deduped.filter((n) => !n.accessible).length;
+    const frameUrls = [...new Set(frameNodes.map((n) => n.url))];
+
+    return {
+      contextTree: deduped,
+      totalContextsFound: deduped.length,
+      maxDepthReached,
+      inaccessibleCount,
+      frameUrls,
+      embeddedAppUrls,
+    };
+  } catch {
+    return empty;
+  }
+}