@detiq/crawler 1.1.13 → 1.1.15

package/src/nested-window-crawler.ts

@@ -0,0 +1,310 @@
+/**
+ * Nested window / browsing-context crawler — discovers and records frames,
+ * embedded apps, and popup windows that are already open in the browser
+ * context at the time of a crawl step.
+ *
+ * Complements multi-context-handler.ts: that module handles proactive popup
+ * interception via context.on('page'); this module snapshots what is already
+ * present in the frame tree and context page list.
+ */
+
+import type { Page, BrowserContext } from 'playwright';
+
+export type BrowsingContextType = 'page' | 'frame' | 'popup' | 'embedded-app';
+
+export interface BrowsingContextNode {
+  id: string;
+  type: BrowsingContextType;
+  url: string;
+  parentId: string | null;
+  depth: number;
+  title: string;
+  interactiveCount: number;
+  accessible: boolean;
+  isEmbeddedApp: boolean;
+}
+
+export interface NestedCrawlResult {
+  contextTree: BrowsingContextNode[];
+  totalContextsFound: number;
+  maxDepthReached: number;
+  inaccessibleCount: number;
+  frameUrls: string[];
+  embeddedAppUrls: string[];
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Deterministic 12-char hex id derived from context type + URL via FNV-1a.
+ * No external deps — algorithm implemented inline.
+ */
+function generateContextId(type: string, url: string): string {
+  let h = 0x811c9dc5;
+  const s = type + url;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = (h * 0x01000193) >>> 0;
+  }
+  return h.toString(16).padStart(8, '0').slice(0, 12);
+}
+
+/**
+ * Returns the frame depth by walking the parentFrame() chain.
+ * The main frame has depth 0.
+ */
+function computeFrameDepth(frame: import('playwright').Frame): number {
+  let depth = 0;
+  let current = frame.parentFrame();
+  while (current !== null) {
+    depth++;
+    current = current.parentFrame();
+  }
+  return depth;
+}
+
+/**
+ * Walks all frames attached to the page and returns a BrowsingContextNode
+ * for each reachable, non-trivial frame up to maxDepth.
+ */
+async function crawlFrameTree(
+  page: any,
+  maxDepth = 4,
+): Promise<BrowsingContextNode[]> {
+  const nodes: BrowsingContextNode[] = [];
+
+  try {
+    const frames = page.frames();
+    const limited = frames.slice(0, 200);
+
+    for (const frame of limited) {
+      try {
+        const url = frame.url();
+
+        // Skip blank / synthetic frames
+        if (
+          !url ||
+          url === 'about:blank' ||
+          url.startsWith('javascript:') ||
+          url.startsWith('data:')
+        ) {
+          continue;
+        }
+
+        const depth = computeFrameDepth(frame);
+        if (depth > maxDepth) continue;
+
+        const parentFrame = frame.parentFrame();
+        const parentId = parentFrame
+          ? generateContextId('frame', parentFrame.url())
+          : null;
+
+        let title = '';
+        let interactiveCount = 0;
+        let accessible = false;
+
+        try {
+          title = await frame.title();
+          interactiveCount = await frame
+            .locator('button, a[href], input, select')
+            .count();
+          accessible = true;
+        } catch {
+          // cross-origin or sandboxed — treat as inaccessible
+        }
+
+        nodes.push({
+          id: generateContextId('frame', url),
+          type: 'frame',
+          url,
+          parentId,
+          depth,
+          title,
+          interactiveCount,
+          accessible,
+          isEmbeddedApp: false,
+        });
+      } catch {
+        // skip individual frame errors
+      }
+    }
+  } catch {
+    // skip entirely if frames() fails
+  }
+
+  return nodes;
+}
+
+/**
+ * Evaluates the page DOM to identify iframes that look like full embedded
+ * applications (large viewport coverage, or app-class/id/src signals).
+ * Returns up to 5 absolute URLs.
+ */
+async function detectEmbeddedApps(page: any): Promise<string[]> {
+  try {
+    const srcs = await (page.evaluate as any)(() => {
+      return Array.from(document.querySelectorAll('iframe'))
+        .filter((f) => {
+          const r = f.getBoundingClientRect();
+          const vw = window.innerWidth;
+          const vh = window.innerHeight;
+          const isLarge = r.width >= vw * 0.7 && r.height >= vh * 0.7;
+          const hasAppClass = /embed|dashboard|app-frame|portal/i.test(
+            f.className + f.id + (f.getAttribute('src') ?? ''),
+          );
+          return isLarge || hasAppClass;
+        })
+        .map((f) => f.src)
+        .filter(Boolean);
+    });
+
+    const baseUrl = page.url();
+    const resolved: string[] = [];
+
+    for (const src of srcs.slice(0, 5)) {
+      try {
+        resolved.push(new URL(src, baseUrl).href);
+      } catch {
+        // skip unparseable src
+      }
+    }
+
+    return resolved;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Snapshots all pages currently open in the context that are not the seed
+ * URL and records them as popup nodes at depth 1.
+ *
+ * Proactive interception (context.on('page')) is handled separately by
+ * attachNewTabListener in multi-context-handler.ts.
+ */
+async function crawlPopupChain(
+  context: BrowserContext,
+  seedUrl: string,
+): Promise<BrowsingContextNode[]> {
+  const nodes: BrowsingContextNode[] = [];
+
+  try {
+    const pages = context.pages();
+
+    for (const pg of pages) {
+      try {
+        const url = pg.url();
+
+        if (!url || url === 'about:blank' || url === seedUrl) continue;
+
+        let title = '';
+        let interactiveCount = 0;
+        let accessible = false;
+
+        try {
+          title = await pg.title();
+          interactiveCount = await pg
+            .locator('button, a[href], input, select')
+            .count();
+          accessible = true;
+        } catch {
+          // page may be closed or cross-origin restricted
+        }
+
+        nodes.push({
+          id: generateContextId('popup', url),
+          type: 'popup',
+          url,
+          parentId: null, // chain depth not reliably trackable for existing pages
+          depth: 1,
+          title,
+          interactiveCount,
+          accessible,
+          isEmbeddedApp: false,
+        });
+      } catch {
+        // skip individual page errors
+      }
+    }
+  } catch {
+    // skip if context.pages() fails
+  }
+
+  return nodes;
+}
+
+// ---------------------------------------------------------------------------
+// Exported entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Discovers all nested browsing contexts reachable from the current page:
+ * sub-frames, embedded applications, and already-open popup pages.
+ *
+ * All failures are non-fatal — errors are caught and an empty result is
+ * returned so the caller's crawl step can continue uninterrupted.
+ */
+export async function crawlNestedContexts(
+  page: any,
+  context: BrowserContext | null,
+  seedUrl: string,
+  opts?: { maxFrameDepth?: number; maxPopupDepth?: number },
+): Promise<NestedCrawlResult> {
+  const empty: NestedCrawlResult = {
+    contextTree: [],
+    totalContextsFound: 0,
+    maxDepthReached: 0,
+    inaccessibleCount: 0,
+    frameUrls: [],
+    embeddedAppUrls: [],
+  };
+
+  try {
+    const maxFrameDepth = opts?.maxFrameDepth ?? 4;
+
+    // Run independent discovery passes concurrently
+    const [frameNodes, embeddedAppUrls, popupNodes] = await Promise.all([
+      crawlFrameTree(page, maxFrameDepth).catch((): BrowsingContextNode[] => []),
+      detectEmbeddedApps(page).catch((): string[] => []),
+      context
+        ? crawlPopupChain(context, seedUrl).catch((): BrowsingContextNode[] => [])
+        : Promise.resolve<BrowsingContextNode[]>([]),
+    ]);
+
+    // Promote frame nodes that match an embedded-app URL
+    const embeddedSet = new Set(embeddedAppUrls);
+    for (const node of frameNodes) {
+      if (embeddedSet.has(node.url)) {
+        node.isEmbeddedApp = true;
+        node.type = 'embedded-app';
+      }
+    }
+
+    const contextTree: BrowsingContextNode[] = [...frameNodes, ...popupNodes];
+
+    // Deduplicate by id (same URL appearing in both lists)
+    const seen = new Set<string>();
+    const deduped = contextTree.filter((n) => {
+      if (seen.has(n.id)) return false;
+      seen.add(n.id);
+      return true;
+    });
+
+    const maxDepthReached = deduped.reduce((max, n) => Math.max(max, n.depth), 0);
+    const inaccessibleCount = deduped.filter((n) => !n.accessible).length;
+    const frameUrls = [...new Set(frameNodes.map((n) => n.url))];
+
+    return {
+      contextTree: deduped,
+      totalContextsFound: deduped.length,
+      maxDepthReached,
+      inaccessibleCount,
+      frameUrls,
+      embeddedAppUrls,
+    };
+  } catch {
+    return empty;
+  }
+}

package/src/network-action-correlator.ts

@@ -0,0 +1,320 @@
+/**
+ * Network-action correlator — captures XHR/fetch requests that occur during
+ * a named UI action (click, form submit, navigation) and maps them to API
+ * endpoint contracts.
+ *
+ * Usage:
+ *   const correlator = createCorrelator();
+ *   const detach = attachNetworkInterceptor(page, correlator);
+ *   correlator.startWindow('Submit login form');
+ *   await page.click('#login-btn');
+ *   const correlation = correlator.endWindow();
+ *   detach();
+ *   const result = correlator.getResult();
+ */
+
+import type { Page, Request, Response } from 'playwright';
+
+export interface NetworkRequestRecord {
+  url: string;
+  urlPattern: string;
+  method: string;
+  bodySchema: Record<string, string> | null;
+  status: number | null;
+  responseSchema: Record<string, string> | null;
+  isGraphQL: boolean;
+  graphQLOperation?: string;
+  resourceType: string;
+  correlatedActionLabel?: string;
+}
+
+export interface ActionCorrelation {
+  actionLabel: string;
+  requests: NetworkRequestRecord[];
+}
+
+export interface NetworkCorrelationResult {
+  correlations: ActionCorrelation[];
+  uniqueEndpoints: string[];
+  graphqlOperations: string[];
+  totalRequestsCaptured: number;
+  coverage: number;
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function extractUrlPattern(url: string): string {
+  try {
+    const u = new URL(url);
+    const path = u.pathname
+      .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, ':uuid')
+      .replace(/\/[0-9a-f]{24}(\/|$)/g, '/:mongoId$1')
+      .replace(/\/\d{6,}(\/|$)/g, '/:id$1')
+      .replace(/\/\d{1,5}(\/|$)/g, '/:id$1');
+    return path;
+  } catch {
+    return url.slice(0, 80);
+  }
+}
+
+function extractBodySchema(bodyText: string): Record<string, string> | null {
+  if (!bodyText || bodyText.length > 50_000) return null;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bodyText);
+  } catch {
+    return null;
+  }
+
+  const schema: Record<string, string> = {};
+  let keyCount = 0;
+
+  function traverse(value: unknown, prefix: string, depth: number): void {
+    if (keyCount >= 30 || depth > 3) return;
+    if (value === null || value === undefined) return;
+
+    if (Array.isArray(value)) {
+      schema[prefix] = 'array';
+      keyCount++;
+      return;
+    }
+
+    if (typeof value === 'object') {
+      for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+        if (keyCount >= 30) break;
+        const key = prefix ? `${prefix}.${k}` : k;
+        if (v === null) {
+          schema[key] = 'null';
+          keyCount++;
+        } else if (Array.isArray(v)) {
+          schema[key] = 'array';
+          keyCount++;
+        } else if (typeof v === 'object') {
+          traverse(v, key, depth + 1);
+        } else {
+          schema[key] = typeof v;
+          keyCount++;
+        }
+      }
+      return;
+    }
+
+    // Primitive at top level
+    if (prefix) {
+      schema[prefix] = typeof value;
+      keyCount++;
+    }
+  }
+
+  traverse(parsed, '', 0);
+  return keyCount > 0 ? schema : null;
+}
+
+const ANALYTICS_DOMAINS = [
+  'google-analytics',
+  'doubleclick',
+  'hotjar',
+  'mixpanel',
+  'segment',
+  'sentry',
+  'datadog',
+  'newrelic',
+  'amplitude',
+  'fullstory',
+];
+
+const API_PATH_PATTERNS = [
+  '/api/',
+  '/graphql',
+  '/v1/',
+  '/v2/',
+  '/rest/',
+  '/query',
+  '/rpc',
+  '/trpc',
+];
+
+const EXCLUDED_RESOURCE_TYPES = new Set(['image', 'font', 'stylesheet', 'media', 'websocket']);
+
+function isApiRequest(url: string, resourceType: string): boolean {
+  if (EXCLUDED_RESOURCE_TYPES.has(resourceType)) return false;
+
+  try {
+    const hostname = new URL(url).hostname;
+    if (ANALYTICS_DOMAINS.some((d) => hostname.includes(d))) return false;
+  } catch {
+    return false;
+  }
+
+  if (resourceType === 'xhr' || resourceType === 'fetch') return true;
+
+  const lower = url.toLowerCase();
+  return API_PATH_PATTERNS.some((p) => lower.includes(p));
+}
+
+// ---------------------------------------------------------------------------
+// Correlator
+// ---------------------------------------------------------------------------
+
+interface NetworkCorrelator {
+  startWindow(actionLabel: string): void;
+  recordRequest(rec: NetworkRequestRecord): void;
+  endWindow(): ActionCorrelation;
+  getResult(): NetworkCorrelationResult;
+}
+
+export function createCorrelator(): NetworkCorrelator {
+  let currentLabel: string | null = null;
+  let currentRequests: NetworkRequestRecord[] = [];
+  const correlations: ActionCorrelation[] = [];
+
+  return {
+    startWindow(actionLabel: string): void {
+      currentLabel = actionLabel;
+      currentRequests = [];
+    },
+
+    recordRequest(rec: NetworkRequestRecord): void {
+      if (currentLabel === null) return;
+      currentRequests.push({ ...rec, correlatedActionLabel: currentLabel });
+    },
+
+    endWindow(): ActionCorrelation {
+      const label = currentLabel ?? '(unknown)';
+      const correlation: ActionCorrelation = {
+        actionLabel: label,
+        requests: currentRequests,
+      };
+      correlations.push(correlation);
+      currentLabel = null;
+      currentRequests = [];
+      return correlation;
+    },
+
+    getResult(): NetworkCorrelationResult {
+      const uniqueEndpoints = [
+        ...new Set(
+          correlations.flatMap((c) => c.requests.map((r) => r.urlPattern)),
+        ),
+      ];
+
+      const graphqlOperations = [
+        ...new Set(
+          correlations
+            .flatMap((c) => c.requests)
+            .filter((r) => r.isGraphQL && r.graphQLOperation)
+            .map((r) => r.graphQLOperation as string),
+        ),
+      ];
+
+      const totalRequestsCaptured = correlations.reduce(
+        (sum, c) => sum + c.requests.length,
+        0,
+      );
+
+      const coverage =
+        correlations.length === 0
+          ? 0
+          : correlations.filter((c) => c.requests.length >= 1).length /
+            correlations.length;
+
+      return {
+        correlations,
+        uniqueEndpoints,
+        graphqlOperations,
+        totalRequestsCaptured,
+        coverage,
+      };
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Page interceptor
+// ---------------------------------------------------------------------------
+
+export function attachNetworkInterceptor(
+  page: Page,
+  correlator: ReturnType<typeof createCorrelator>,
+): () => void {
+  const pendingRequests = new Map<string, { bodyText: string; resourceType: string }>();
+
+  const onRequest = (req: Request): void => {
+    const url = req.url();
+    const resourceType = req.resourceType();
+    if (!isApiRequest(url, resourceType)) return;
+    let bodyText = '';
+    try {
+      bodyText = req.postData() ?? '';
+    } catch {
+      // postData() may throw on non-POST requests in some Playwright versions
+    }
+    pendingRequests.set(url, { bodyText, resourceType });
+  };
+
+  const onResponse = async (res: Response): Promise<void> => {
+    const url = res.url();
+    const pending = pendingRequests.get(url);
+    if (!pending) return;
+    pendingRequests.delete(url);
+
+    const method = res.request().method();
+
+    let bodySchema: Record<string, string> | null = null;
+    try {
+      bodySchema = extractBodySchema(pending.bodyText);
+    } catch {
+      // ignore schema extraction failures
+    }
+
+    let responseSchema: Record<string, string> | null = null;
+    let isGraphQL = false;
+    let graphQLOperation: string | undefined;
+
+    const contentType = res.headers()['content-type'] ?? '';
+    if (contentType.includes('application/json')) {
+      try {
+        const respText = await res.text();
+        if (respText.length < 20_000) {
+          responseSchema = extractBodySchema(respText);
+        }
+      } catch {
+        // response body may no longer be available
+      }
+    }
+
+    // Detect GraphQL from request body
+    try {
+      const bodyParsed = JSON.parse(pending.bodyText) as Record<string, unknown>;
+      if (typeof bodyParsed?.operationName === 'string') {
+        isGraphQL = true;
+        graphQLOperation = bodyParsed.operationName;
+      }
+    } catch {
+      // not JSON or no operationName
+    }
+
+    const rec: NetworkRequestRecord = {
+      url,
+      urlPattern: extractUrlPattern(url),
+      method,
+      bodySchema,
+      status: res.status(),
+      responseSchema,
+      isGraphQL,
+      graphQLOperation,
+      resourceType: pending.resourceType,
+    };
+    correlator.recordRequest(rec);
+  };
+
+  page.on('request', onRequest);
+  page.on('response', onResponse);
+
+  return () => {
+    page.off('request', onRequest);
+    page.off('response', onResponse);
+  };
+}