@detiq/crawler 1.1.14 → 1.1.15

package/src/network-action-correlator.ts

@@ -0,0 +1,320 @@
+/**
+ * Network-action correlator — captures XHR/fetch requests that occur during
+ * a named UI action (click, form submit, navigation) and maps them to API
+ * endpoint contracts.
+ *
+ * Usage:
+ *   const correlator = createCorrelator();
+ *   const detach = attachNetworkInterceptor(page, correlator);
+ *   correlator.startWindow('Submit login form');
+ *   await page.click('#login-btn');
+ *   const correlation = correlator.endWindow();
+ *   detach();
+ *   const result = correlator.getResult();
+ */
+
+import type { Page, Request, Response } from 'playwright';
+
+export interface NetworkRequestRecord {
+  url: string;
+  urlPattern: string;
+  method: string;
+  bodySchema: Record<string, string> | null;
+  status: number | null;
+  responseSchema: Record<string, string> | null;
+  isGraphQL: boolean;
+  graphQLOperation?: string;
+  resourceType: string;
+  correlatedActionLabel?: string;
+}
+
+export interface ActionCorrelation {
+  actionLabel: string;
+  requests: NetworkRequestRecord[];
+}
+
+export interface NetworkCorrelationResult {
+  correlations: ActionCorrelation[];
+  uniqueEndpoints: string[];
+  graphqlOperations: string[];
+  totalRequestsCaptured: number;
+  coverage: number;
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function extractUrlPattern(url: string): string {
+  try {
+    const u = new URL(url);
+    const path = u.pathname
+      .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, ':uuid')
+      .replace(/\/[0-9a-f]{24}(\/|$)/g, '/:mongoId$1')
+      .replace(/\/\d{6,}(\/|$)/g, '/:id$1')
+      .replace(/\/\d{1,5}(\/|$)/g, '/:id$1');
+    return path;
+  } catch {
+    return url.slice(0, 80);
+  }
+}
+
+function extractBodySchema(bodyText: string): Record<string, string> | null {
+  if (!bodyText || bodyText.length > 50_000) return null;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bodyText);
+  } catch {
+    return null;
+  }
+
+  const schema: Record<string, string> = {};
+  let keyCount = 0;
+
+  function traverse(value: unknown, prefix: string, depth: number): void {
+    if (keyCount >= 30 || depth > 3) return;
+    if (value === null || value === undefined) return;
+
+    if (Array.isArray(value)) {
+      schema[prefix] = 'array';
+      keyCount++;
+      return;
+    }
+
+    if (typeof value === 'object') {
+      for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+        if (keyCount >= 30) break;
+        const key = prefix ? `${prefix}.${k}` : k;
+        if (v === null) {
+          schema[key] = 'null';
+          keyCount++;
+        } else if (Array.isArray(v)) {
+          schema[key] = 'array';
+          keyCount++;
+        } else if (typeof v === 'object') {
+          traverse(v, key, depth + 1);
+        } else {
+          schema[key] = typeof v;
+          keyCount++;
+        }
+      }
+      return;
+    }
+
+    // Primitive at top level
+    if (prefix) {
+      schema[prefix] = typeof value;
+      keyCount++;
+    }
+  }
+
+  traverse(parsed, '', 0);
+  return keyCount > 0 ? schema : null;
+}
+
+const ANALYTICS_DOMAINS = [
+  'google-analytics',
+  'doubleclick',
+  'hotjar',
+  'mixpanel',
+  'segment',
+  'sentry',
+  'datadog',
+  'newrelic',
+  'amplitude',
+  'fullstory',
+];
+
+const API_PATH_PATTERNS = [
+  '/api/',
+  '/graphql',
+  '/v1/',
+  '/v2/',
+  '/rest/',
+  '/query',
+  '/rpc',
+  '/trpc',
+];
+
+const EXCLUDED_RESOURCE_TYPES = new Set(['image', 'font', 'stylesheet', 'media', 'websocket']);
+
+function isApiRequest(url: string, resourceType: string): boolean {
+  if (EXCLUDED_RESOURCE_TYPES.has(resourceType)) return false;
+
+  try {
+    const hostname = new URL(url).hostname;
+    if (ANALYTICS_DOMAINS.some((d) => hostname.includes(d))) return false;
+  } catch {
+    return false;
+  }
+
+  if (resourceType === 'xhr' || resourceType === 'fetch') return true;
+
+  const lower = url.toLowerCase();
+  return API_PATH_PATTERNS.some((p) => lower.includes(p));
+}
+
+// ---------------------------------------------------------------------------
+// Correlator
+// ---------------------------------------------------------------------------
+
+interface NetworkCorrelator {
+  startWindow(actionLabel: string): void;
+  recordRequest(rec: NetworkRequestRecord): void;
+  endWindow(): ActionCorrelation;
+  getResult(): NetworkCorrelationResult;
+}
+
+export function createCorrelator(): NetworkCorrelator {
+  let currentLabel: string | null = null;
+  let currentRequests: NetworkRequestRecord[] = [];
+  const correlations: ActionCorrelation[] = [];
+
+  return {
+    startWindow(actionLabel: string): void {
+      currentLabel = actionLabel;
+      currentRequests = [];
+    },
+
+    recordRequest(rec: NetworkRequestRecord): void {
+      if (currentLabel === null) return;
+      currentRequests.push({ ...rec, correlatedActionLabel: currentLabel });
+    },
+
+    endWindow(): ActionCorrelation {
+      const label = currentLabel ?? '(unknown)';
+      const correlation: ActionCorrelation = {
+        actionLabel: label,
+        requests: currentRequests,
+      };
+      correlations.push(correlation);
+      currentLabel = null;
+      currentRequests = [];
+      return correlation;
+    },
+
+    getResult(): NetworkCorrelationResult {
+      const uniqueEndpoints = [
+        ...new Set(
+          correlations.flatMap((c) => c.requests.map((r) => r.urlPattern)),
+        ),
+      ];
+
+      const graphqlOperations = [
+        ...new Set(
+          correlations
+            .flatMap((c) => c.requests)
+            .filter((r) => r.isGraphQL && r.graphQLOperation)
+            .map((r) => r.graphQLOperation as string),
+        ),
+      ];
+
+      const totalRequestsCaptured = correlations.reduce(
+        (sum, c) => sum + c.requests.length,
+        0,
+      );
+
+      const coverage =
+        correlations.length === 0
+          ? 0
+          : correlations.filter((c) => c.requests.length >= 1).length /
+            correlations.length;
+
+      return {
+        correlations,
+        uniqueEndpoints,
+        graphqlOperations,
+        totalRequestsCaptured,
+        coverage,
+      };
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Page interceptor
+// ---------------------------------------------------------------------------
+
+export function attachNetworkInterceptor(
+  page: Page,
+  correlator: ReturnType<typeof createCorrelator>,
+): () => void {
+  const pendingRequests = new Map<string, { bodyText: string; resourceType: string }>();
+
+  const onRequest = (req: Request): void => {
+    const url = req.url();
+    const resourceType = req.resourceType();
+    if (!isApiRequest(url, resourceType)) return;
+    let bodyText = '';
+    try {
+      bodyText = req.postData() ?? '';
+    } catch {
+      // postData() may throw on non-POST requests in some Playwright versions
+    }
+    pendingRequests.set(url, { bodyText, resourceType });
+  };
+
+  const onResponse = async (res: Response): Promise<void> => {
+    const url = res.url();
+    const pending = pendingRequests.get(url);
+    if (!pending) return;
+    pendingRequests.delete(url);
+
+    const method = res.request().method();
+
+    let bodySchema: Record<string, string> | null = null;
+    try {
+      bodySchema = extractBodySchema(pending.bodyText);
+    } catch {
+      // ignore schema extraction failures
+    }
+
+    let responseSchema: Record<string, string> | null = null;
+    let isGraphQL = false;
+    let graphQLOperation: string | undefined;
+
+    const contentType = res.headers()['content-type'] ?? '';
+    if (contentType.includes('application/json')) {
+      try {
+        const respText = await res.text();
+        if (respText.length < 20_000) {
+          responseSchema = extractBodySchema(respText);
+        }
+      } catch {
+        // response body may no longer be available
+      }
+    }
+
+    // Detect GraphQL from request body
+    try {
+      const bodyParsed = JSON.parse(pending.bodyText) as Record<string, unknown>;
+      if (typeof bodyParsed?.operationName === 'string') {
+        isGraphQL = true;
+        graphQLOperation = bodyParsed.operationName;
+      }
+    } catch {
+      // not JSON or no operationName
+    }
+
+    const rec: NetworkRequestRecord = {
+      url,
+      urlPattern: extractUrlPattern(url),
+      method,
+      bodySchema,
+      status: res.status(),
+      responseSchema,
+      isGraphQL,
+      graphQLOperation,
+      resourceType: pending.resourceType,
+    };
+    correlator.recordRequest(rec);
+  };
+
+  page.on('request', onRequest);
+  page.on('response', onResponse);
+
+  return () => {
+    page.off('request', onRequest);
+    page.off('response', onResponse);
+  };
+}

package/src/performance-metrics-collector.ts

@@ -0,0 +1,243 @@
+/**
+ * Performance metrics collector — captures Core Web Vitals and navigation timing
+ * for pages visited during crawl runs.
+ * Collects LCP, CLS, INP, TTFB, long tasks, and slow resources via PerformanceObserver
+ * and the Navigation/Resource Timing APIs.
+ */
+
+import type { Page } from 'playwright';
+
+export type CwvScore = 'good' | 'needs-improvement' | 'poor';
+
+export interface LcpMetric {
+  value: number;
+  score: CwvScore;
+  elementTag: string | null;
+  url: string | null;
+}
+
+export interface ClsMetric {
+  value: number;
+  score: CwvScore;
+  shiftCount: number;
+}
+
+export interface InpMetric {
+  p75: number;
+  score: CwvScore;
+  sampleCount: number;
+}
+
+export interface NavTimingMetric {
+  ttfb: number;
+  ttfbScore: CwvScore;
+  domContentLoaded: number;
+  domComplete: number;
+  loadComplete: number;
+  transferSizeBytes: number;
+  protocol: string;
+}
+
+export interface SlowResource {
+  path: string;
+  durationMs: number;
+  transferSizeBytes: number;
+  initiatorType: string;
+}
+
+export interface PerformanceMetricsResult {
+  lcp: LcpMetric | null;
+  cls: ClsMetric | null;
+  inp: InpMetric | null;
+  navTiming: NavTimingMetric | null;
+  slowResources: SlowResource[];
+  longTaskCount: number;
+  captureSuccess: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Observer injection — call before page navigation via addInitScript
+// ---------------------------------------------------------------------------
+
+export async function injectPerformanceObservers(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    (window as any).__zetaPerf = { lcp: null, cls: 0, clsCount: 0, inp: [], longTaskCount: 0 };
+
+    try {
+      new PerformanceObserver(list => {
+        const entries = list.getEntries();
+        const last = entries[entries.length - 1] as any;
+        if (last) {
+          (window as any).__zetaPerf.lcp = {
+            value: last.startTime,
+            elementTag: last.element?.tagName || null,
+            url: last.url || null,
+          };
+        }
+      }).observe({ entryTypes: ['largest-contentful-paint'] });
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        for (const entry of list.getEntries() as any[]) {
+          if (!entry.hadRecentInput) {
+            (window as any).__zetaPerf.cls += entry.value;
+            (window as any).__zetaPerf.clsCount++;
+          }
+        }
+      }).observe({ entryTypes: ['layout-shift'] });
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        for (const entry of list.getEntries() as any[]) {
+          (window as any).__zetaPerf.inp.push(entry.duration);
+        }
+      }).observe({ entryTypes: ['event'], durationThreshold: 40 } as any);
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        (window as any).__zetaPerf.longTaskCount += list.getEntries().length;
+      }).observe({ entryTypes: ['longtask'] });
+    } catch {}
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Scoring helpers
+// ---------------------------------------------------------------------------
+
+function scoreLcp(value: number): CwvScore {
+  if (value <= 2500) return 'good';
+  if (value <= 4000) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreCls(value: number): CwvScore {
+  if (value <= 0.1) return 'good';
+  if (value <= 0.25) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreInp(p75: number): CwvScore {
+  if (p75 <= 200) return 'good';
+  if (p75 <= 500) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreTtfb(value: number): CwvScore {
+  if (value <= 800) return 'good';
+  if (value <= 1800) return 'needs-improvement';
+  return 'poor';
+}
+
+// ---------------------------------------------------------------------------
+// Internal collector
+// ---------------------------------------------------------------------------
+
+async function collectMetrics(page: Page): Promise<PerformanceMetricsResult> {
+  const rawMetrics = await page.evaluate(() => {
+    const p = (window as any).__zetaPerf || {};
+    const nav = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming | undefined;
+    const resources = performance.getEntriesByType('resource')
+      .filter(r => r.duration > 500)
+      .sort((a, b) => b.duration - a.duration)
+      .slice(0, 10)
+      .map(r => ({
+        path: (() => { try { return new URL(r.name).pathname; } catch { return r.name.slice(0, 80); } })(),
+        durationMs: Math.round(r.duration),
+        transferSizeBytes: (r as any).transferSize || 0,
+        initiatorType: (r as any).initiatorType || '',
+      }));
+    const inpValues = p.inp || [];
+    const p75 = inpValues.length > 0
+      ? inpValues.sort((a: number, b: number) => a - b)[Math.floor(inpValues.length * 0.75)]
+      : 0;
+    return {
+      lcp: p.lcp || null,
+      cls: p.cls ?? null,
+      clsCount: p.clsCount ?? 0,
+      inpP75: p75,
+      inpCount: inpValues.length,
+      longTaskCount: p.longTaskCount || 0,
+      nav: nav ? {
+        ttfb: Math.round(nav.responseStart - nav.requestStart),
+        domContentLoaded: Math.round(nav.domContentLoadedEventEnd - nav.startTime),
+        domComplete: Math.round(nav.domComplete - nav.startTime),
+        loadComplete: Math.round(nav.loadEventEnd - nav.startTime),
+        transferSizeBytes: (nav as any).transferSize || 0,
+        protocol: (nav as any).nextHopProtocol || '',
+      } : null,
+      resources,
+    };
+  });
+
+  const lcp: LcpMetric | null = rawMetrics.lcp
+    ? {
+        value: rawMetrics.lcp.value,
+        score: scoreLcp(rawMetrics.lcp.value),
+        elementTag: rawMetrics.lcp.elementTag,
+        url: rawMetrics.lcp.url,
+      }
+    : null;
+
+  const cls: ClsMetric | null = rawMetrics.cls !== null
+    ? {
+        value: rawMetrics.cls,
+        score: scoreCls(rawMetrics.cls),
+        shiftCount: rawMetrics.clsCount,
+      }
+    : null;
+
+  const inp: InpMetric | null = rawMetrics.inpCount > 0
+    ? {
+        p75: rawMetrics.inpP75,
+        score: scoreInp(rawMetrics.inpP75),
+        sampleCount: rawMetrics.inpCount,
+      }
+    : null;
+
+  const navTiming: NavTimingMetric | null = rawMetrics.nav
+    ? {
+        ttfb: rawMetrics.nav.ttfb,
+        ttfbScore: scoreTtfb(rawMetrics.nav.ttfb),
+        domContentLoaded: rawMetrics.nav.domContentLoaded,
+        domComplete: rawMetrics.nav.domComplete,
+        loadComplete: rawMetrics.nav.loadComplete,
+        transferSizeBytes: rawMetrics.nav.transferSizeBytes,
+        protocol: rawMetrics.nav.protocol,
+      }
+    : null;
+
+  return {
+    lcp,
+    cls,
+    inp,
+    navTiming,
+    slowResources: rawMetrics.resources,
+    longTaskCount: rawMetrics.longTaskCount,
+    captureSuccess: true,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+const emptyResult: Omit<PerformanceMetricsResult, 'captureSuccess'> = {
+  lcp: null,
+  cls: null,
+  inp: null,
+  navTiming: null,
+  slowResources: [],
+  longTaskCount: 0,
+};
+
+export async function collectPagePerformance(page: Page): Promise<PerformanceMetricsResult> {
+  try {
+    return await collectMetrics(page);
+  } catch {
+    return { ...emptyResult, captureSuccess: false };
+  }
+}

package/src/performance-metrics.ts

@@ -40,7 +40,7 @@ export interface PagePerfMetrics {
   measuredAt: string;
 }
 
-export async function collectPerfMetrics(page: Page, url: string): Promise<PagePerfMetrics> {
+export async function collectPerfMetrics(page: any, url: string): Promise<PagePerfMetrics> {
   const measuredAt = new Date().toISOString();
 
   // Inject PerformanceObserver for LCP + CLS, wait up to 3s

package/src/screen-reader.ts

@@ -15,8 +15,11 @@ export interface RawElement {
   expectedData?: string;
 }
 
-export async function readScreen(page: Page): Promise<RawElement[]> {
+export async function readScreen(page: any): Promise<RawElement[]> {
   return page.evaluate(() => {
+    // esbuild injects __name() for named arrow-function constants (e.g. `const priority = ...`).
+    // Polyfill prevents ReferenceError when the serialized callback runs in the browser.
+    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
     const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = [];
     const seen = new Set<string>();