@detiq/crawler 1.1.14 → 1.1.15

package/src/service-worker-interceptor.ts

@@ -0,0 +1,185 @@
+/**
+ * Service Worker detection — inspects SW registrations, cache storage, background
+ * sync queues, and PWA installability for a crawled page.
+ * Surfaces offline-first contracts and push-capable surfaces that HTTP-only
+ * crawlers cannot observe.
+ */
+
+import type { Page } from 'playwright';
+
+export interface SwRegistration {
+  scope: string;
+  scriptURL: string | null;
+  state: 'active' | 'installing' | 'waiting' | 'none';
+  updateViaCache: string;
+}
+
+export interface SwCacheEntry {
+  cacheName: string;
+  entryCount: number;
+  sampleUrls: string[];
+}
+
+export interface BackgroundSyncInfo {
+  scope: string;
+  pendingTags: string[];
+}
+
+export interface PwaInfo {
+  isInstallable: boolean;
+  manifestUrl: string | null;
+  hasPushPermission: boolean;
+}
+
+export interface ServiceWorkerResult {
+  supported: boolean;
+  active: boolean;
+  registrations: SwRegistration[];
+  cacheStorages: SwCacheEntry[];
+  backgroundSync: BackgroundSyncInfo[];
+  pwa: PwaInfo;
+  totalCachedEntries: number;
+}
+
+const EMPTY_RESULT: ServiceWorkerResult = {
+  supported: false,
+  active: false,
+  registrations: [],
+  cacheStorages: [],
+  backgroundSync: [],
+  pwa: { isInstallable: false, manifestUrl: null, hasPushPermission: false },
+  totalCachedEntries: 0,
+};
+
+async function detectServiceWorkers(
+  page: any,
+): Promise<{ supported: boolean; active: boolean; registrations: SwRegistration[] }> {
+  const result = await page.evaluate(async () => {
+    if (!('serviceWorker' in navigator)) return { supported: false, active: false, registrations: [] };
+    try {
+      const controller = navigator.serviceWorker.controller;
+      const regs = await navigator.serviceWorker.getRegistrations();
+      return {
+        supported: true,
+        active: !!controller,
+        registrations: regs.map(r => ({
+          scope: r.scope,
+          scriptURL: r.active?.scriptURL || r.installing?.scriptURL || r.waiting?.scriptURL || null,
+          state: r.active ? 'active' : r.installing ? 'installing' : r.waiting ? 'waiting' : 'none',
+          updateViaCache: r.updateViaCache,
+        })),
+      };
+    } catch {
+      return { supported: true, active: false, registrations: [] };
+    }
+  });
+  return result as { supported: boolean; active: boolean; registrations: SwRegistration[] };
+}
+
+async function enumerateCacheStorage(page: any): Promise<SwCacheEntry[]> {
+  const caches = await page.evaluate(async () => {
+    if (!('caches' in window)) return [];
+    try {
+      const names = await window.caches.keys();
+      const result = [];
+      for (const name of names) {
+        try {
+          const cache = await window.caches.open(name);
+          const keys = await cache.keys();
+          result.push({
+            cacheName: name,
+            entryCount: keys.length,
+            sampleUrls: keys.slice(0, 15).map(k => {
+              try { return new URL(k.url).pathname; } catch { return k.url.slice(0, 80); }
+            }),
+          });
+        } catch { /* skip inaccessible cache */ }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return caches as SwCacheEntry[];
+}
+
+async function detectBackgroundSync(page: any): Promise<BackgroundSyncInfo[]> {
+  const syncInfo = await page.evaluate(async () => {
+    if (!('serviceWorker' in navigator)) return [];
+    try {
+      const regs = await navigator.serviceWorker.getRegistrations();
+      const result = [];
+      for (const reg of regs) {
+        if ('sync' in reg) {
+          try {
+            const tags = await (reg as any).sync.getTags();
+            result.push({ scope: reg.scope, pendingTags: tags || [] });
+          } catch {
+            result.push({ scope: reg.scope, pendingTags: [] });
+          }
+        }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return syncInfo as BackgroundSyncInfo[];
+}
+
+async function detectPwa(page: any): Promise<PwaInfo> {
+  const pwa = await page.evaluate(() => {
+    const manifestEl = document.querySelector('link[rel="manifest"]');
+    const manifestUrl = manifestEl?.getAttribute('href') || null;
+    const hasSW = 'serviceWorker' in navigator && !!navigator.serviceWorker.controller;
+    let hasPushPermission = false;
+    try { hasPushPermission = Notification.permission === 'granted'; } catch { /* ignore */ }
+    return {
+      isInstallable: !!(manifestUrl && hasSW),
+      manifestUrl,
+      hasPushPermission,
+    };
+  });
+  return pwa as PwaInfo;
+}
+
+export async function inspectServiceWorker(page: any): Promise<ServiceWorkerResult> {
+  try {
+    // Allow SW on HTTPS or localhost (http://localhost and http://127.0.0.1 support SW)
+    const url = page.url();
+    const isSecureContext = url.startsWith('https://') ||
+      /^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?/.test(url);
+    if (!isSecureContext) return { ...EMPTY_RESULT };
+
+    const [swResult, cacheResult, syncResult, pwaResult] = await Promise.allSettled([
+      detectServiceWorkers(page),
+      enumerateCacheStorage(page),
+      detectBackgroundSync(page),
+      detectPwa(page),
+    ]);
+
+    const sw = swResult.status === 'fulfilled'
+      ? swResult.value
+      : { supported: false, active: false, registrations: [] };
+
+    const cacheStorages = cacheResult.status === 'fulfilled' ? cacheResult.value : [];
+    const backgroundSync = syncResult.status === 'fulfilled' ? syncResult.value : [];
+    const pwa = pwaResult.status === 'fulfilled'
+      ? pwaResult.value
+      : EMPTY_RESULT.pwa;
+
+    const totalCachedEntries = cacheStorages.reduce((sum, c) => sum + c.entryCount, 0);
+
+    return {
+      supported: sw.supported,
+      active: sw.active,
+      registrations: sw.registrations,
+      cacheStorages,
+      backgroundSync,
+      pwa,
+      totalCachedEntries,
+    };
+  } catch {
+    return { ...EMPTY_RESULT };
+  }
+}

package/src/session-capture.ts

@@ -207,6 +207,7 @@ export async function validateAndCaptureSession(
   },
   llmConfig?: { modelName: string; apiKey: string },
   captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },
+  loginInstructions?: string,
 ): Promise<{ storageState: string; cookies: number; localStorage: number; finalUrl?: string }> {
   const browser = await chromium.launch({ headless: true });
   try {
@@ -257,6 +258,8 @@ export async function validateAndCaptureSession(
           { username: creds.username, password: creds.password, getOneTimeCode },
           appUrl,
           captchaOpts,
+          undefined,
+          loginInstructions,
         );
         if (shLoggedIn) {
           const shCtx = (sh as any).context ?? (sh as any).browserContext;

package/src/shadow-dom-crawler.ts

@@ -0,0 +1,198 @@
+/**
+ * Shadow DOM crawler — enumerate shadow hosts and detect web component libraries.
+ *
+ * Walks the live DOM recursively (up to depth 8) to find all open shadow roots
+ * and custom elements with closed shadow roots, then classifies the component
+ * framework in use. Results feed ZeTa's test generator so it can account for
+ * elements that are invisible to standard Playwright locators.
+ */
+
+import type { Page } from 'playwright';
+
+export interface ShadowInteractiveEl {
+  role: string;
+  label: string;
+  type: string;
+  tagName: string;
+}
+
+export interface ShadowHostInfo {
+  tagName: string;
+  hostId: string;
+  shadowMode: 'open' | 'closed';
+  depth: number;
+  interactiveCount: number;
+  interactiveElements: ShadowInteractiveEl[];
+  slotNames: string[];
+  childShadowHosts: number;
+}
+
+export interface ShadowDomCrawlResult {
+  shadowHostCount: number;
+  openShadowCount: number;
+  closedShadowCount: number;
+  maxNestingDepth: number;
+  hosts: ShadowHostInfo[];
+  totalInteractiveInShadow: number;
+  detectedLibraries: string[];
+  blindSpots: string[];
+}
+
+interface RawShadowHost {
+  tagName: string;
+  hostId: string;
+  shadowMode: 'open' | 'closed';
+  depth: number;
+  interactiveCount: number;
+  interactiveDetails: Array<{ role: string; label: string; type: string; tagName: string }>;
+  slotNames: string[];
+  childShadowHosts: number;
+}
+
+async function enumerateShadowHosts(page: any): Promise<ShadowHostInfo[]> {
+  const raw: RawShadowHost[] = await page.evaluate(() => {
+    // esbuild injects __name() for named function declarations; polyfill prevents ReferenceError in browser.
+    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
+    function collectHosts(root: Document | ShadowRoot, depth: number): RawShadowHost[] {
+      if (depth > 8) return [];
+      const hosts: RawShadowHost[] = [];
+      const els = root.querySelectorAll('*');
+      for (const el of Array.from(els)) {
+        const htmlEl = el as HTMLElement & { shadowRoot: ShadowRoot | null };
+        if (htmlEl.shadowRoot) {
+          const interactive = htmlEl.shadowRoot.querySelectorAll(
+            'button,input,select,textarea,a[href],[role="button"],[role="link"],[role="checkbox"],[role="menuitem"],[role="tab"],[role="switch"]'
+          );
+          const slots = Array.from(htmlEl.shadowRoot.querySelectorAll('slot')).map(
+            (s) => (s as HTMLSlotElement).name || 'default'
+          );
+          const nestedHosts = Array.from(htmlEl.shadowRoot.querySelectorAll('*')).filter(
+            (c) => (c as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot
+          ).length;
+          hosts.push({
+            tagName: htmlEl.tagName.toLowerCase(),
+            hostId:
+              htmlEl.id ||
+              (htmlEl.className?.toString()?.split(' ')[0]) ||
+              '',
+            shadowMode: 'open',
+            depth,
+            interactiveCount: interactive.length,
+            interactiveDetails: Array.from(interactive)
+              .slice(0, 8)
+              .map((i) => {
+                const iEl = i as HTMLElement;
+                return {
+                  role: iEl.getAttribute('role') || iEl.tagName.toLowerCase(),
+                  label: (
+                    iEl.getAttribute('aria-label') ||
+                    iEl.getAttribute('placeholder') ||
+                    iEl.textContent?.trim() ||
+                    ''
+                  ).slice(0, 40),
+                  type: iEl.getAttribute('type') || '',
+                  tagName: iEl.tagName.toLowerCase(),
+                };
+              }),
+            slotNames: slots,
+            childShadowHosts: nestedHosts,
+          });
+          const nested = collectHosts(htmlEl.shadowRoot, depth + 1);
+          hosts.push(...nested);
+        } else if (el.tagName.includes('-') && !(el as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot) {
+          hosts.push({
+            tagName: el.tagName.toLowerCase(),
+            hostId: (el as HTMLElement).id || '',
+            shadowMode: 'closed',
+            depth,
+            interactiveCount: 0,
+            interactiveDetails: [],
+            slotNames: [],
+            childShadowHosts: 0,
+          });
+        }
+      }
+      return hosts;
+    }
+    return collectHosts(document, 0);
+  });
+
+  return raw.slice(0, 100).map((h) => ({
+    tagName: h.tagName,
+    hostId: h.hostId,
+    shadowMode: h.shadowMode,
+    depth: h.depth,
+    interactiveCount: h.interactiveCount,
+    interactiveElements: h.interactiveDetails,
+    slotNames: h.slotNames,
+    childShadowHosts: h.childShadowHosts,
+  }));
+}
+
+async function detectWebComponentLibrary(page: any): Promise<string[]> {
+  return page.evaluate(() => {
+    const checks: [string, string][] = [
+      ['sl-button', 'Shoelace'],
+      ['fast-button', 'Microsoft FAST'],
+      ['mwc-button', 'Material Web Components'],
+      ['ion-button', 'Ionic'],
+      ['vaadin-button', 'Vaadin'],
+      ['fluent-button', 'Fluent UI Web Components'],
+      ['hy-icon', 'Hydra'],
+    ];
+    const detected: string[] = [];
+    for (const [tag, name] of checks) {
+      if (customElements.get(tag)) detected.push(name);
+    }
+    const allTags = Array.from(document.querySelectorAll('*'))
+      .map((e) => e.tagName.toLowerCase())
+      .filter((t) => t.includes('-'));
+    const uniqueTags = Array.from(new Set(allTags));
+    if (uniqueTags.some((t) => t.startsWith('sl-')) && !detected.includes('Shoelace'))
+      detected.push('Shoelace');
+    if (uniqueTags.some((t) => t.startsWith('fast-')) && !detected.includes('Microsoft FAST'))
+      detected.push('Microsoft FAST');
+    if (uniqueTags.some((t) => t.startsWith('ion-')) && !detected.includes('Ionic'))
+      detected.push('Ionic');
+    return detected;
+  });
+}
+
+export async function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult> {
+  const empty: ShadowDomCrawlResult = {
+    shadowHostCount: 0,
+    openShadowCount: 0,
+    closedShadowCount: 0,
+    maxNestingDepth: 0,
+    hosts: [],
+    totalInteractiveInShadow: 0,
+    detectedLibraries: [],
+    blindSpots: [],
+  };
+
+  try {
+    const [hosts, detectedLibraries] = await Promise.all([
+      enumerateShadowHosts(page),
+      detectWebComponentLibrary(page),
+    ]);
+
+    const openHosts = hosts.filter((h) => h.shadowMode === 'open');
+    const closedHosts = hosts.filter((h) => h.shadowMode === 'closed');
+    const maxNestingDepth = hosts.reduce((max, h) => Math.max(max, h.depth), 0);
+    const totalInteractiveInShadow = openHosts.reduce((sum, h) => sum + h.interactiveCount, 0);
+    const blindSpots = [...new Set(closedHosts.map((h) => h.tagName))];
+
+    return {
+      shadowHostCount: hosts.length,
+      openShadowCount: openHosts.length,
+      closedShadowCount: closedHosts.length,
+      maxNestingDepth,
+      hosts,
+      totalInteractiveInShadow,
+      detectedLibraries,
+      blindSpots,
+    };
+  } catch {
+    return empty;
+  }
+}

package/src/state-graph.ts

@@ -0,0 +1,280 @@
+import type { Page } from 'playwright';
+
+// ---------------------------------------------------------------------------
+// Internal: FNV-1a 32-bit SimHash
+// ---------------------------------------------------------------------------
+
+function fnv1a32(token: string): number {
+  let h = 2166136261;
+  for (let i = 0; i < token.length; i++) {
+    h = Math.imul(h ^ token.charCodeAt(i), 16777619);
+    h = h >>> 0;
+  }
+  return h;
+}
+
+export function simHash32(text: string): number {
+  const tokens = text
+    .split(/\W+/)
+    .filter((t) => t.length > 2)
+    .slice(0, 500);
+
+  const vec = new Array<number>(32).fill(0);
+
+  for (const token of tokens) {
+    const h = fnv1a32(token);
+    for (let b = 0; b < 32; b++) {
+      vec[b] += (h >>> b) & 1 ? 1 : -1;
+    }
+  }
+
+  let result = 0;
+  for (let b = 0; b < 32; b++) {
+    if (vec[b] > 0) result |= 1 << b;
+  }
+  return result >>> 0;
+}
+
+export function hammingDistance32(a: number, b: number): number {
+  let n = (a ^ b) >>> 0;
+  n = n - ((n >>> 1) & 0x55555555);
+  n = (n & 0x33333333) + ((n >>> 2) & 0x33333333);
+  n = (n + (n >>> 4)) & 0x0f0f0f0f;
+  return Math.imul(n, 0x01010101) >>> 24;
+}
+
+// ---------------------------------------------------------------------------
+// URL normalisation
+// ---------------------------------------------------------------------------
+
+const STRIP_PARAMS = new Set([
+  'utm_source', 'utm_medium', 'utm_campaign',
+  'ref', '_ga', 'fbclid', 'session_id',
+]);
+
+export function normalizeUrl(raw: string): string {
+  try {
+    const u = new URL(raw);
+    u.hash = '';
+    const keep: Array<[string, string]> = [];
+    u.searchParams.forEach((v, k) => {
+      if (!STRIP_PARAMS.has(k)) keep.push([k, v]);
+    });
+    keep.sort((a, b) => a[0].localeCompare(b[0]));
+    u.search = keep.length
+      ? '?' + keep.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&')
+      : '';
+    return u.toString();
+  } catch {
+    return raw;
+  }
+}
+
+export function computeStateId(normalizedUrl: string, hash: number): string {
+  let path = normalizedUrl.slice(0, 40).replace(/[^a-zA-Z0-9]/g, '_');
+  const hex = hash.toString(16).padStart(8, '0');
+  return `${path}_${hex}`;
+}
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+export interface UIStateNode {
+  id: string;
+  url: string;
+  normalizedUrl: string;
+  title: string;
+  interactiveElementCount: number;
+  domFingerprint: string;
+  simHashValue: number;
+  visitedAt: string;
+}
+
+export interface StateTransition {
+  fromId: string;
+  toId: string;
+  action: string;
+  label: string;
+  triggeredAt: string;
+}
+
+export interface CoverageReport {
+  totalStates: number;
+  totalTransitions: number;
+  uniqueUrls: number;
+  interactiveElementsTotal: number;
+  estimatedCoveragePercent: number;
+}
+
+// ---------------------------------------------------------------------------
+// Page fingerprinting
+// ---------------------------------------------------------------------------
+
+export async function fingerprintPage(
+  page: Page,
+  visitedAt: string,
+): Promise<{
+  fingerprint: string;
+  visibleText: string;
+  interactiveCount: number;
+  simHashValue: number;
+}> {
+  let visibleText = '';
+  let interactiveCount = 0;
+
+  try {
+    visibleText = await page.evaluate(() => {
+      const el = document.querySelector('main') ?? document.body;
+      return (el as HTMLElement).innerText ?? '';
+    });
+  } catch {
+    visibleText = '';
+  }
+
+  try {
+    interactiveCount = await page.evaluate(() =>
+      document.querySelectorAll(
+        'a[href], button, input, select, textarea, [role="button"], [role="link"], [tabindex]',
+      ).length,
+    );
+  } catch {
+    interactiveCount = 0;
+  }
+
+  const simHashValue = simHash32(visibleText + visitedAt.slice(0, 10));
+  const fingerprint = simHashValue.toString(16).padStart(8, '0');
+
+  return { fingerprint, visibleText, interactiveCount, simHashValue };
+}
+
+// ---------------------------------------------------------------------------
+// ScreenTransitionGraph
+// ---------------------------------------------------------------------------
+
+export class ScreenTransitionGraph {
+  private states = new Map<string, UIStateNode>();
+  private transitions: StateTransition[] = [];
+  /** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
+  private hashIndex = new Map<number, string[]>();
+
+  // -------------------------------------------------------------------------
+  // State management
+  // -------------------------------------------------------------------------
+
+  addState(node: UIStateNode): { added: boolean; duplicateOf?: string } {
+    if (this.states.has(node.id)) {
+      return { added: false, duplicateOf: node.id };
+    }
+
+    // Check near-duplicate via hamming distance across all indexed hashes
+    for (const [indexedHash, ids] of this.hashIndex) {
+      if (hammingDistance32(node.simHashValue, indexedHash) < 3) {
+        return { added: false, duplicateOf: ids[0] };
+      }
+    }
+
+    this.states.set(node.id, node);
+
+    const bucket = this.hashIndex.get(node.simHashValue);
+    if (bucket) {
+      bucket.push(node.id);
+    } else {
+      this.hashIndex.set(node.simHashValue, [node.id]);
+    }
+
+    return { added: true };
+  }
+
+  addTransition(
+    fromId: string,
+    toId: string,
+    action: string,
+    label: string,
+    triggeredAt: string,
+  ): void {
+    this.transitions.push({ fromId, toId, action, label, triggeredAt });
+  }
+
+  isDuplicate(url: string, visibleText: string, threshold = 3): boolean {
+    const normalized = normalizeUrl(url);
+    const hash = simHash32(visibleText);
+    for (const [indexedHash] of this.hashIndex) {
+      if (hammingDistance32(hash, indexedHash) < threshold) return true;
+    }
+    // Also check exact normalized URL match
+    for (const node of this.states.values()) {
+      if (node.normalizedUrl === normalized) return true;
+    }
+    return false;
+  }
+
+  // -------------------------------------------------------------------------
+  // Reporting
+  // -------------------------------------------------------------------------
+
+  getCoverageReport(): CoverageReport {
+    const totalStates = this.states.size;
+    const totalTransitions = this.transitions.length;
+    const uniqueUrls = new Set(
+      Array.from(this.states.values()).map((n) => n.normalizedUrl),
+    ).size;
+    const interactiveElementsTotal = Array.from(this.states.values()).reduce(
+      (sum, n) => sum + n.interactiveElementCount,
+      0,
+    );
+    const estimatedCoveragePercent = Math.min(
+      100,
+      (totalTransitions / Math.max(1, totalStates)) * 50,
+    );
+    return {
+      totalStates,
+      totalTransitions,
+      uniqueUrls,
+      interactiveElementsTotal,
+      estimatedCoveragePercent,
+    };
+  }
+
+  // -------------------------------------------------------------------------
+  // Serialisation
+  // -------------------------------------------------------------------------
+
+  toJSON(): { states: UIStateNode[]; transitions: StateTransition[] } {
+    return {
+      states: Array.from(this.states.values()),
+      transitions: this.transitions,
+    };
+  }
+
+  static fromJSON(data: { states: UIStateNode[]; transitions: StateTransition[] }): ScreenTransitionGraph {
+    const graph = new ScreenTransitionGraph();
+    for (const node of data.states) {
+      graph.states.set(node.id, node);
+      const bucket = graph.hashIndex.get(node.simHashValue);
+      if (bucket) {
+        bucket.push(node.id);
+      } else {
+        graph.hashIndex.set(node.simHashValue, [node.id]);
+      }
+    }
+    graph.transitions = [...data.transitions];
+    return graph;
+  }
+
+  // -------------------------------------------------------------------------
+  // Accessors
+  // -------------------------------------------------------------------------
+
+  getState(id: string): UIStateNode | undefined {
+    return this.states.get(id);
+  }
+
+  getAllStates(): UIStateNode[] {
+    return Array.from(this.states.values());
+  }
+
+  getTransitionsFrom(id: string): StateTransition[] {
+    return this.transitions.filter((t) => t.fromId === id);
+  }
+}