@detiq/crawler 1.1.13 → 1.1.15

package/src/performance-metrics-collector.ts

@@ -0,0 +1,243 @@
+/**
+ * Performance metrics collector — captures Core Web Vitals and navigation timing
+ * for pages visited during crawl runs.
+ * Collects LCP, CLS, INP, TTFB, long tasks, and slow resources via PerformanceObserver
+ * and the Navigation/Resource Timing APIs.
+ */
+
+import type { Page } from 'playwright';
+
+export type CwvScore = 'good' | 'needs-improvement' | 'poor';
+
+export interface LcpMetric {
+  value: number;
+  score: CwvScore;
+  elementTag: string | null;
+  url: string | null;
+}
+
+export interface ClsMetric {
+  value: number;
+  score: CwvScore;
+  shiftCount: number;
+}
+
+export interface InpMetric {
+  p75: number;
+  score: CwvScore;
+  sampleCount: number;
+}
+
+export interface NavTimingMetric {
+  ttfb: number;
+  ttfbScore: CwvScore;
+  domContentLoaded: number;
+  domComplete: number;
+  loadComplete: number;
+  transferSizeBytes: number;
+  protocol: string;
+}
+
+export interface SlowResource {
+  path: string;
+  durationMs: number;
+  transferSizeBytes: number;
+  initiatorType: string;
+}
+
+export interface PerformanceMetricsResult {
+  lcp: LcpMetric | null;
+  cls: ClsMetric | null;
+  inp: InpMetric | null;
+  navTiming: NavTimingMetric | null;
+  slowResources: SlowResource[];
+  longTaskCount: number;
+  captureSuccess: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Observer injection — call before page navigation via addInitScript
+// ---------------------------------------------------------------------------
+
+export async function injectPerformanceObservers(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    (window as any).__zetaPerf = { lcp: null, cls: 0, clsCount: 0, inp: [], longTaskCount: 0 };
+
+    try {
+      new PerformanceObserver(list => {
+        const entries = list.getEntries();
+        const last = entries[entries.length - 1] as any;
+        if (last) {
+          (window as any).__zetaPerf.lcp = {
+            value: last.startTime,
+            elementTag: last.element?.tagName || null,
+            url: last.url || null,
+          };
+        }
+      }).observe({ entryTypes: ['largest-contentful-paint'] });
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        for (const entry of list.getEntries() as any[]) {
+          if (!entry.hadRecentInput) {
+            (window as any).__zetaPerf.cls += entry.value;
+            (window as any).__zetaPerf.clsCount++;
+          }
+        }
+      }).observe({ entryTypes: ['layout-shift'] });
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        for (const entry of list.getEntries() as any[]) {
+          (window as any).__zetaPerf.inp.push(entry.duration);
+        }
+      }).observe({ entryTypes: ['event'], durationThreshold: 40 } as any);
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        (window as any).__zetaPerf.longTaskCount += list.getEntries().length;
+      }).observe({ entryTypes: ['longtask'] });
+    } catch {}
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Scoring helpers
+// ---------------------------------------------------------------------------
+
+function scoreLcp(value: number): CwvScore {
+  if (value <= 2500) return 'good';
+  if (value <= 4000) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreCls(value: number): CwvScore {
+  if (value <= 0.1) return 'good';
+  if (value <= 0.25) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreInp(p75: number): CwvScore {
+  if (p75 <= 200) return 'good';
+  if (p75 <= 500) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreTtfb(value: number): CwvScore {
+  if (value <= 800) return 'good';
+  if (value <= 1800) return 'needs-improvement';
+  return 'poor';
+}
+
+// ---------------------------------------------------------------------------
+// Internal collector
+// ---------------------------------------------------------------------------
+
+async function collectMetrics(page: Page): Promise<PerformanceMetricsResult> {
+  const rawMetrics = await page.evaluate(() => {
+    const p = (window as any).__zetaPerf || {};
+    const nav = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming | undefined;
+    const resources = performance.getEntriesByType('resource')
+      .filter(r => r.duration > 500)
+      .sort((a, b) => b.duration - a.duration)
+      .slice(0, 10)
+      .map(r => ({
+        path: (() => { try { return new URL(r.name).pathname; } catch { return r.name.slice(0, 80); } })(),
+        durationMs: Math.round(r.duration),
+        transferSizeBytes: (r as any).transferSize || 0,
+        initiatorType: (r as any).initiatorType || '',
+      }));
+    const inpValues = p.inp || [];
+    const p75 = inpValues.length > 0
+      ? inpValues.sort((a: number, b: number) => a - b)[Math.floor(inpValues.length * 0.75)]
+      : 0;
+    return {
+      lcp: p.lcp || null,
+      cls: p.cls ?? null,
+      clsCount: p.clsCount ?? 0,
+      inpP75: p75,
+      inpCount: inpValues.length,
+      longTaskCount: p.longTaskCount || 0,
+      nav: nav ? {
+        ttfb: Math.round(nav.responseStart - nav.requestStart),
+        domContentLoaded: Math.round(nav.domContentLoadedEventEnd - nav.startTime),
+        domComplete: Math.round(nav.domComplete - nav.startTime),
+        loadComplete: Math.round(nav.loadEventEnd - nav.startTime),
+        transferSizeBytes: (nav as any).transferSize || 0,
+        protocol: (nav as any).nextHopProtocol || '',
+      } : null,
+      resources,
+    };
+  });
+
+  const lcp: LcpMetric | null = rawMetrics.lcp
+    ? {
+        value: rawMetrics.lcp.value,
+        score: scoreLcp(rawMetrics.lcp.value),
+        elementTag: rawMetrics.lcp.elementTag,
+        url: rawMetrics.lcp.url,
+      }
+    : null;
+
+  const cls: ClsMetric | null = rawMetrics.cls !== null
+    ? {
+        value: rawMetrics.cls,
+        score: scoreCls(rawMetrics.cls),
+        shiftCount: rawMetrics.clsCount,
+      }
+    : null;
+
+  const inp: InpMetric | null = rawMetrics.inpCount > 0
+    ? {
+        p75: rawMetrics.inpP75,
+        score: scoreInp(rawMetrics.inpP75),
+        sampleCount: rawMetrics.inpCount,
+      }
+    : null;
+
+  const navTiming: NavTimingMetric | null = rawMetrics.nav
+    ? {
+        ttfb: rawMetrics.nav.ttfb,
+        ttfbScore: scoreTtfb(rawMetrics.nav.ttfb),
+        domContentLoaded: rawMetrics.nav.domContentLoaded,
+        domComplete: rawMetrics.nav.domComplete,
+        loadComplete: rawMetrics.nav.loadComplete,
+        transferSizeBytes: rawMetrics.nav.transferSizeBytes,
+        protocol: rawMetrics.nav.protocol,
+      }
+    : null;
+
+  return {
+    lcp,
+    cls,
+    inp,
+    navTiming,
+    slowResources: rawMetrics.resources,
+    longTaskCount: rawMetrics.longTaskCount,
+    captureSuccess: true,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+const emptyResult: Omit<PerformanceMetricsResult, 'captureSuccess'> = {
+  lcp: null,
+  cls: null,
+  inp: null,
+  navTiming: null,
+  slowResources: [],
+  longTaskCount: 0,
+};
+
+export async function collectPagePerformance(page: Page): Promise<PerformanceMetricsResult> {
+  try {
+    return await collectMetrics(page);
+  } catch {
+    return { ...emptyResult, captureSuccess: false };
+  }
+}

package/src/performance-metrics.ts

@@ -40,7 +40,7 @@ export interface PagePerfMetrics {
   measuredAt: string;
 }
 
-export async function collectPerfMetrics(page: Page, url: string): Promise<PagePerfMetrics> {
+export async function collectPerfMetrics(page: any, url: string): Promise<PagePerfMetrics> {
   const measuredAt = new Date().toISOString();
 
   // Inject PerformanceObserver for LCP + CLS, wait up to 3s

package/src/screen-reader.ts

@@ -15,8 +15,11 @@ export interface RawElement {
   expectedData?: string;
 }
 
-export async function readScreen(page: Page): Promise<RawElement[]> {
+export async function readScreen(page: any): Promise<RawElement[]> {
   return page.evaluate(() => {
+    // esbuild injects __name() for named arrow-function constants (e.g. `const priority = ...`).
+    // Polyfill prevents ReferenceError when the serialized callback runs in the browser.
+    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
     const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = [];
     const seen = new Set<string>();
 

package/src/service-worker-interceptor.ts

@@ -0,0 +1,185 @@
+/**
+ * Service Worker detection — inspects SW registrations, cache storage, background
+ * sync queues, and PWA installability for a crawled page.
+ * Surfaces offline-first contracts and push-capable surfaces that HTTP-only
+ * crawlers cannot observe.
+ */
+
+import type { Page } from 'playwright';
+
+export interface SwRegistration {
+  scope: string;
+  scriptURL: string | null;
+  state: 'active' | 'installing' | 'waiting' | 'none';
+  updateViaCache: string;
+}
+
+export interface SwCacheEntry {
+  cacheName: string;
+  entryCount: number;
+  sampleUrls: string[];
+}
+
+export interface BackgroundSyncInfo {
+  scope: string;
+  pendingTags: string[];
+}
+
+export interface PwaInfo {
+  isInstallable: boolean;
+  manifestUrl: string | null;
+  hasPushPermission: boolean;
+}
+
+export interface ServiceWorkerResult {
+  supported: boolean;
+  active: boolean;
+  registrations: SwRegistration[];
+  cacheStorages: SwCacheEntry[];
+  backgroundSync: BackgroundSyncInfo[];
+  pwa: PwaInfo;
+  totalCachedEntries: number;
+}
+
+const EMPTY_RESULT: ServiceWorkerResult = {
+  supported: false,
+  active: false,
+  registrations: [],
+  cacheStorages: [],
+  backgroundSync: [],
+  pwa: { isInstallable: false, manifestUrl: null, hasPushPermission: false },
+  totalCachedEntries: 0,
+};
+
+async function detectServiceWorkers(
+  page: any,
+): Promise<{ supported: boolean; active: boolean; registrations: SwRegistration[] }> {
+  const result = await page.evaluate(async () => {
+    if (!('serviceWorker' in navigator)) return { supported: false, active: false, registrations: [] };
+    try {
+      const controller = navigator.serviceWorker.controller;
+      const regs = await navigator.serviceWorker.getRegistrations();
+      return {
+        supported: true,
+        active: !!controller,
+        registrations: regs.map(r => ({
+          scope: r.scope,
+          scriptURL: r.active?.scriptURL || r.installing?.scriptURL || r.waiting?.scriptURL || null,
+          state: r.active ? 'active' : r.installing ? 'installing' : r.waiting ? 'waiting' : 'none',
+          updateViaCache: r.updateViaCache,
+        })),
+      };
+    } catch {
+      return { supported: true, active: false, registrations: [] };
+    }
+  });
+  return result as { supported: boolean; active: boolean; registrations: SwRegistration[] };
+}
+
+async function enumerateCacheStorage(page: any): Promise<SwCacheEntry[]> {
+  const caches = await page.evaluate(async () => {
+    if (!('caches' in window)) return [];
+    try {
+      const names = await window.caches.keys();
+      const result = [];
+      for (const name of names) {
+        try {
+          const cache = await window.caches.open(name);
+          const keys = await cache.keys();
+          result.push({
+            cacheName: name,
+            entryCount: keys.length,
+            sampleUrls: keys.slice(0, 15).map(k => {
+              try { return new URL(k.url).pathname; } catch { return k.url.slice(0, 80); }
+            }),
+          });
+        } catch { /* skip inaccessible cache */ }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return caches as SwCacheEntry[];
+}
+
+async function detectBackgroundSync(page: any): Promise<BackgroundSyncInfo[]> {
+  const syncInfo = await page.evaluate(async () => {
+    if (!('serviceWorker' in navigator)) return [];
+    try {
+      const regs = await navigator.serviceWorker.getRegistrations();
+      const result = [];
+      for (const reg of regs) {
+        if ('sync' in reg) {
+          try {
+            const tags = await (reg as any).sync.getTags();
+            result.push({ scope: reg.scope, pendingTags: tags || [] });
+          } catch {
+            result.push({ scope: reg.scope, pendingTags: [] });
+          }
+        }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return syncInfo as BackgroundSyncInfo[];
+}
+
+async function detectPwa(page: any): Promise<PwaInfo> {
+  const pwa = await page.evaluate(() => {
+    const manifestEl = document.querySelector('link[rel="manifest"]');
+    const manifestUrl = manifestEl?.getAttribute('href') || null;
+    const hasSW = 'serviceWorker' in navigator && !!navigator.serviceWorker.controller;
+    let hasPushPermission = false;
+    try { hasPushPermission = Notification.permission === 'granted'; } catch { /* ignore */ }
+    return {
+      isInstallable: !!(manifestUrl && hasSW),
+      manifestUrl,
+      hasPushPermission,
+    };
+  });
+  return pwa as PwaInfo;
+}
+
+export async function inspectServiceWorker(page: any): Promise<ServiceWorkerResult> {
+  try {
+    // Allow SW on HTTPS or localhost (http://localhost and http://127.0.0.1 support SW)
+    const url = page.url();
+    const isSecureContext = url.startsWith('https://') ||
+      /^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?/.test(url);
+    if (!isSecureContext) return { ...EMPTY_RESULT };
+
+    const [swResult, cacheResult, syncResult, pwaResult] = await Promise.allSettled([
+      detectServiceWorkers(page),
+      enumerateCacheStorage(page),
+      detectBackgroundSync(page),
+      detectPwa(page),
+    ]);
+
+    const sw = swResult.status === 'fulfilled'
+      ? swResult.value
+      : { supported: false, active: false, registrations: [] };
+
+    const cacheStorages = cacheResult.status === 'fulfilled' ? cacheResult.value : [];
+    const backgroundSync = syncResult.status === 'fulfilled' ? syncResult.value : [];
+    const pwa = pwaResult.status === 'fulfilled'
+      ? pwaResult.value
+      : EMPTY_RESULT.pwa;
+
+    const totalCachedEntries = cacheStorages.reduce((sum, c) => sum + c.entryCount, 0);
+
+    return {
+      supported: sw.supported,
+      active: sw.active,
+      registrations: sw.registrations,
+      cacheStorages,
+      backgroundSync,
+      pwa,
+      totalCachedEntries,
+    };
+  } catch {
+    return { ...EMPTY_RESULT };
+  }
+}

package/src/session-capture.ts

@@ -207,6 +207,7 @@ export async function validateAndCaptureSession(
   },
   llmConfig?: { modelName: string; apiKey: string },
   captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },
+  loginInstructions?: string,
 ): Promise<{ storageState: string; cookies: number; localStorage: number; finalUrl?: string }> {
   const browser = await chromium.launch({ headless: true });
   try {
@@ -257,6 +258,8 @@ export async function validateAndCaptureSession(
           { username: creds.username, password: creds.password, getOneTimeCode },
           appUrl,
           captchaOpts,
+          undefined,
+          loginInstructions,
         );
         if (shLoggedIn) {
           const shCtx = (sh as any).context ?? (sh as any).browserContext;

package/src/shadow-dom-crawler.ts

@@ -0,0 +1,198 @@
+/**
+ * Shadow DOM crawler — enumerate shadow hosts and detect web component libraries.
+ *
+ * Walks the live DOM recursively (up to depth 8) to find all open shadow roots
+ * and custom elements with closed shadow roots, then classifies the component
+ * framework in use. Results feed ZeTa's test generator so it can account for
+ * elements that are invisible to standard Playwright locators.
+ */
+
+import type { Page } from 'playwright';
+
+export interface ShadowInteractiveEl {
+  role: string;
+  label: string;
+  type: string;
+  tagName: string;
+}
+
+export interface ShadowHostInfo {
+  tagName: string;
+  hostId: string;
+  shadowMode: 'open' | 'closed';
+  depth: number;
+  interactiveCount: number;
+  interactiveElements: ShadowInteractiveEl[];
+  slotNames: string[];
+  childShadowHosts: number;
+}
+
+export interface ShadowDomCrawlResult {
+  shadowHostCount: number;
+  openShadowCount: number;
+  closedShadowCount: number;
+  maxNestingDepth: number;
+  hosts: ShadowHostInfo[];
+  totalInteractiveInShadow: number;
+  detectedLibraries: string[];
+  blindSpots: string[];
+}
+
+interface RawShadowHost {
+  tagName: string;
+  hostId: string;
+  shadowMode: 'open' | 'closed';
+  depth: number;
+  interactiveCount: number;
+  interactiveDetails: Array<{ role: string; label: string; type: string; tagName: string }>;
+  slotNames: string[];
+  childShadowHosts: number;
+}
+
+async function enumerateShadowHosts(page: any): Promise<ShadowHostInfo[]> {
+  const raw: RawShadowHost[] = await page.evaluate(() => {
+    // esbuild injects __name() for named function declarations; polyfill prevents ReferenceError in browser.
+    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
+    function collectHosts(root: Document | ShadowRoot, depth: number): RawShadowHost[] {
+      if (depth > 8) return [];
+      const hosts: RawShadowHost[] = [];
+      const els = root.querySelectorAll('*');
+      for (const el of Array.from(els)) {
+        const htmlEl = el as HTMLElement & { shadowRoot: ShadowRoot | null };
+        if (htmlEl.shadowRoot) {
+          const interactive = htmlEl.shadowRoot.querySelectorAll(
+            'button,input,select,textarea,a[href],[role="button"],[role="link"],[role="checkbox"],[role="menuitem"],[role="tab"],[role="switch"]'
+          );
+          const slots = Array.from(htmlEl.shadowRoot.querySelectorAll('slot')).map(
+            (s) => (s as HTMLSlotElement).name || 'default'
+          );
+          const nestedHosts = Array.from(htmlEl.shadowRoot.querySelectorAll('*')).filter(
+            (c) => (c as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot
+          ).length;
+          hosts.push({
+            tagName: htmlEl.tagName.toLowerCase(),
+            hostId:
+              htmlEl.id ||
+              (htmlEl.className?.toString()?.split(' ')[0]) ||
+              '',
+            shadowMode: 'open',
+            depth,
+            interactiveCount: interactive.length,
+            interactiveDetails: Array.from(interactive)
+              .slice(0, 8)
+              .map((i) => {
+                const iEl = i as HTMLElement;
+                return {
+                  role: iEl.getAttribute('role') || iEl.tagName.toLowerCase(),
+                  label: (
+                    iEl.getAttribute('aria-label') ||
+                    iEl.getAttribute('placeholder') ||
+                    iEl.textContent?.trim() ||
+                    ''
+                  ).slice(0, 40),
+                  type: iEl.getAttribute('type') || '',
+                  tagName: iEl.tagName.toLowerCase(),
+                };
+              }),
+            slotNames: slots,
+            childShadowHosts: nestedHosts,
+          });
+          const nested = collectHosts(htmlEl.shadowRoot, depth + 1);
+          hosts.push(...nested);
+        } else if (el.tagName.includes('-') && !(el as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot) {
+          hosts.push({
+            tagName: el.tagName.toLowerCase(),
+            hostId: (el as HTMLElement).id || '',
+            shadowMode: 'closed',
+            depth,
+            interactiveCount: 0,
+            interactiveDetails: [],
+            slotNames: [],
+            childShadowHosts: 0,
+          });
+        }
+      }
+      return hosts;
+    }
+    return collectHosts(document, 0);
+  });
+
+  return raw.slice(0, 100).map((h) => ({
+    tagName: h.tagName,
+    hostId: h.hostId,
+    shadowMode: h.shadowMode,
+    depth: h.depth,
+    interactiveCount: h.interactiveCount,
+    interactiveElements: h.interactiveDetails,
+    slotNames: h.slotNames,
+    childShadowHosts: h.childShadowHosts,
+  }));
+}
+
+async function detectWebComponentLibrary(page: any): Promise<string[]> {
+  return page.evaluate(() => {
+    const checks: [string, string][] = [
+      ['sl-button', 'Shoelace'],
+      ['fast-button', 'Microsoft FAST'],
+      ['mwc-button', 'Material Web Components'],
+      ['ion-button', 'Ionic'],
+      ['vaadin-button', 'Vaadin'],
+      ['fluent-button', 'Fluent UI Web Components'],
+      ['hy-icon', 'Hydra'],
+    ];
+    const detected: string[] = [];
+    for (const [tag, name] of checks) {
+      if (customElements.get(tag)) detected.push(name);
+    }
+    const allTags = Array.from(document.querySelectorAll('*'))
+      .map((e) => e.tagName.toLowerCase())
+      .filter((t) => t.includes('-'));
+    const uniqueTags = Array.from(new Set(allTags));
+    if (uniqueTags.some((t) => t.startsWith('sl-')) && !detected.includes('Shoelace'))
+      detected.push('Shoelace');
+    if (uniqueTags.some((t) => t.startsWith('fast-')) && !detected.includes('Microsoft FAST'))
+      detected.push('Microsoft FAST');
+    if (uniqueTags.some((t) => t.startsWith('ion-')) && !detected.includes('Ionic'))
+      detected.push('Ionic');
+    return detected;
+  });
+}
+
+export async function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult> {
+  const empty: ShadowDomCrawlResult = {
+    shadowHostCount: 0,
+    openShadowCount: 0,
+    closedShadowCount: 0,
+    maxNestingDepth: 0,
+    hosts: [],
+    totalInteractiveInShadow: 0,
+    detectedLibraries: [],
+    blindSpots: [],
+  };
+
+  try {
+    const [hosts, detectedLibraries] = await Promise.all([
+      enumerateShadowHosts(page),
+      detectWebComponentLibrary(page),
+    ]);
+
+    const openHosts = hosts.filter((h) => h.shadowMode === 'open');
+    const closedHosts = hosts.filter((h) => h.shadowMode === 'closed');
+    const maxNestingDepth = hosts.reduce((max, h) => Math.max(max, h.depth), 0);
+    const totalInteractiveInShadow = openHosts.reduce((sum, h) => sum + h.interactiveCount, 0);
+    const blindSpots = [...new Set(closedHosts.map((h) => h.tagName))];
+
+    return {
+      shadowHostCount: hosts.length,
+      openShadowCount: openHosts.length,
+      closedShadowCount: closedHosts.length,
+      maxNestingDepth,
+      hosts,
+      totalInteractiveInShadow,
+      detectedLibraries,
+      blindSpots,
+    };
+  } catch {
+    return empty;
+  }
+}