@detiq/crawler 1.1.13 → 1.1.15

package/src/http-engine.ts

@@ -70,6 +70,18 @@ const SKIP_PATH = _skipPathExtra.length > 0
   ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')
   : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
 
+async function getHttpConfig() {
+  const { getSystemConfig } = await import('@detiq/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return {
+    retryLimit:   parseInt(String(cfg?.httpRetryLimit    ?? '5'),     10),
+    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs  ?? '60000'), 10),
+    retry5xx:     parseInt(String(cfg?.http5xxRetryLimit ?? '3'),     10),
+    timeoutMs:    parseInt(String(cfg?.httpTimeoutMs     ?? '15000'), 10),
+    concurrency:  parseInt(String(cfg?.httpConcurrency   ?? '5'),     10),
+  };
+}
+
 export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
 
 export interface HttpScreen {
@@ -101,7 +113,7 @@ export interface HttpCrawlOpts {
   urlFilter?: string;             // regex pattern — only crawl matching URLs
   traversalStrategy?: 'BFS' | 'DFS'; // default BFS
   outputFormats?: HttpOutputFormat[];
-  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'
+  cacheDir?: string;              // default '/tmp/zeta-cache'
   userAgent?: string;
   headers?: Record<string, string>;
   respectRobotsTxt?: boolean;     // default true
@@ -200,8 +212,10 @@ async function fetchWithBackoff(
   url: string,
   cache: ResponseCache,
   opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },
-  retryCount = 0
+  retryCount = 0,
+  httpCfg?: Awaited<ReturnType<typeof getHttpConfig>>
 ): Promise<FetchResult> {
+  if (!httpCfg) httpCfg = await getHttpConfig();
   const t0 = performance.now();
 
   if (cache.isFresh(url)) {
@@ -255,20 +269,20 @@ async function fetchWithBackoff(
 
   // 429 / 503 — exponential backoff with jitter
   if (response.status === 429 || response.status === 503) {
-    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);
     const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
-    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));
+    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(httpCfg.backoffCapMs, 1_000 * Math.pow(2, retryCount));
     const jitter = Math.random() * 1000;
-    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);
     await new Promise(r => setTimeout(r, baseDelay + jitter));
-    return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
   }
 
   // 5xx — retry up to 3 times
   if (response.status >= 500) {
-    if (retryCount < 3) {
+    if (retryCount < httpCfg.retry5xx) {
       await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
-      return fetchWithBackoff(url, cache, opts, retryCount + 1);
+      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
     }
     if (cache.staleIfError(url)) {
       const cached = cache.get(url)!;
@@ -292,11 +306,11 @@ async function fetchWithBackoff(
 
 export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
   const startTime = performance.now();
-  const maxScreens = opts.maxScreens ?? (parseInt(process.env.CRAWLER_HTTP_ENGINE_MAX_SCREENS ?? '') || 100);
-  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
-  const timeoutMs = opts.timeoutMs ?? 15_000;
-  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
-  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
+  const httpCfg = await getHttpConfig();
+  const maxScreens = opts.maxScreens ?? 100;
+  const cacheDir = opts.cacheDir ?? '/tmp/zeta-cache';
+  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;
+  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
   const cache = new ResponseCache(cacheDir);
   const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
   const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;
@@ -340,7 +354,7 @@ export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
 
       let fetchResult: FetchResult;
       try {
-        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);
         if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
       } catch (err: any) {
         console.error(`[http-engine] error fetching ${url}: ${err.message}`);
@@ -399,7 +413,7 @@ export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
     }));
   };
 
-  const concurrency = opts.concurrency ?? 5;
+  const concurrency = opts.concurrency ?? httpCfg.concurrency;
 
   while (queue.length > 0 && screens.length < maxScreens) {
     const batch = queue.splice(0, concurrency);

package/src/index.ts

@@ -35,5 +35,18 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
-export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';
 export type { PlaywrightLocator } from './locator-extractor.js';
+export * from './dropdown-enumerator.js';
+export * from './crud-interaction.js';
+export * from './filter-pagination-crawler.js';
+export * from './multi-context-handler.js';
+export * from './advanced-ui-probes.js';
+export * from './state-graph.js';
+export * from './nested-window-crawler.js';
+export * from './shadow-dom-crawler.js';
+export * from './network-action-correlator.js';
+export * from './crawl-session-manager.js';
+export * from './accessibility-tree-crawler.js';
+export * from './performance-metrics.js';
+export * from './service-worker-interceptor.js';

package/src/job-queue.ts

@@ -1703,7 +1703,9 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
     const appUrl: string = config.appUrl ?? crawlCfg.appUrl;
     if (!appUrl) throw new Error('No app URL configured. Save the crawl settings first.');
 
-    const maxDiscoverUrls: number = config.maxDiscoverUrls ?? 2000;
+    const { getSystemConfig } = await import('@detiq/core');
+    const crawlTuning = await getSystemConfig('crawl_tuning');
+    const maxDiscoverUrls: number = config.maxDiscoverUrls ?? parseInt(String(crawlTuning?.maxDiscoverUrls ?? '2000'), 10);
     const proxyUrl: string | undefined = config.proxyUrl ?? crawlCfg.proxyUrl;
     const screenshotDir = os.tmpdir();
     const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);

package/src/locator-extractor.ts

@@ -220,9 +220,35 @@ export function elementToPlaywrightLocator(element: {
   return null;
 }
 
+// Patterns that indicate crawl-analysis meta-text stored as element records (not real UI elements).
+// These come from accessibility-auditor, screen-reader analysis, or CSS inspector annotations.
+const META_ANNOTATION_RE = /^(css animations|scroll effects detected|design tokens?\s*:|font imports?:|contrast ratio|aria-label audit|landmark|wcag|color contrast|background.*foreground.*card|--[\w-]+,\s*--)/i;
+
+// Auth-wall element patterns — if >50% of locators match these, the page was an auth wall.
+const AUTH_WALL_TEXT_RE = /sign\s*in|log\s*in|platform access required|please sign in|access required|authentication required|session expired|unauthorized/i;
+
+/**
+ * Returns true if an element's text content is crawl-analysis meta-text, not a real UI element.
+ * These are annotations stored by the accessibility auditor, not actual DOM elements.
+ */
+function isMetaAnnotation(el: { role?: string | null; meaning?: string | null; text?: string | null }): boolean {
+  const t = (el.meaning ?? el.text ?? '').trim();
+  if (!t) return false;
+  // Filter crawl-analysis annotations
+  if (META_ANNOTATION_RE.test(t)) return true;
+  // Filter very long text strings with no interactive role — these are paragraph/body copy,
+  // not useful UI locators. Threshold: 120 chars with no testId/label/role
+  const interactiveRoles = new Set(['button', 'link', 'textbox', 'combobox', 'checkbox', 'radio', 'tab', 'option', 'menuitem']);
+  if (t.length > 120 && !interactiveRoles.has((el.role ?? '').toLowerCase())) return true;
+  // Filter CSS variable dumps (contain multiple --var-name patterns)
+  if ((t.match(/--[\w-]+/g) ?? []).length > 3) return true;
+  return false;
+}
+
 /**
  * Convert an array of DB Element records to deduplicated Playwright locators.
  * Deduplicates by key — higher confidence wins.
+ * Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.
  */
 export function elementsToPlaywrightLocators(elements: Array<{
   role?: string | null;
@@ -237,6 +263,9 @@ export function elementsToPlaywrightLocators(elements: Array<{
   const byKey = new Map<string, PlaywrightLocator>();
 
   for (const el of elements) {
+    // Skip crawl-analysis meta-annotations — these are not real page elements
+    if (isMetaAnnotation(el)) continue;
+
     const loc = elementToPlaywrightLocator(el);
     if (!loc) continue;
     const existing = byKey.get(loc.key);
@@ -248,6 +277,22 @@ export function elementsToPlaywrightLocators(elements: Array<{
   return Array.from(byKey.values());
 }
 
+/**
+ * Detect whether a set of locators was extracted from an auth-wall page.
+ * Returns a warning message if auth-wall detected, null otherwise.
+ * Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.
+ */
+export function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null {
+  if (locators.length === 0) return null;
+  const lowCount = locators.filter(l => l.confidence === 'low').length;
+  const hasAuthText = locators.some(l => AUTH_WALL_TEXT_RE.test(l.description));
+  const highCount = locators.filter(l => l.confidence === 'high').length;
+  if (hasAuthText && highCount === 0 && lowCount / locators.length > 0.5) {
+    return 'These locators appear to be from a login/access wall, not the target page. The page requires authentication. Add credentials in Project Settings and re-crawl to get accurate locators for the protected content.';
+  }
+  return null;
+}
+
 /**
  * Format locators as a block for injection into LLM script generation prompts.
  * Returns empty string if no locators.

package/src/multi-context-handler.ts

@@ -0,0 +1,296 @@
+/**
+ * Multi-context browser handling — detects and records new tabs, iframes,
+ * popups, and file downloads triggered during a page crawl.
+ * This lets ZeTa discover cross-origin flows, OAuth redirects, embedded
+ * third-party content, and downloadable resources that single-page HTTP
+ * crawlers would miss entirely.
+ */
+
+import type { Page, BrowserContext } from 'playwright';
+
+export interface NewTabRecord {
+  url: string;
+  linkText: string;
+  isInternal: boolean;
+  httpStatus?: number;
+}
+
+export interface IframeRecord {
+  src: string;
+  crossOrigin: boolean;
+  sandboxed: boolean;
+  sandboxValue: string | null;
+  accessible: boolean;
+  contentSample: string;
+  hasForm: boolean;
+}
+
+export interface PopupRecord {
+  url: string;
+  triggerLabel: string;
+  isOAuthLike: boolean;
+}
+
+export interface DownloadRecord {
+  filename: string;
+  downloadUrl: string;
+  triggerLabel: string;
+}
+
+export interface MultiContextResult {
+  newTabs: NewTabRecord[];
+  iframes: IframeRecord[];
+  popups: PopupRecord[];
+  downloads: DownloadRecord[];
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+async function enumerateIframes(
+  page: Page,
+  seedUrl: string,
+): Promise<IframeRecord[]> {
+  const records: IframeRecord[] = [];
+  try {
+    const seedHost = new URL(seedUrl).hostname;
+    const iframeLocators = await page.locator('iframe').all();
+    const limited = iframeLocators.slice(0, 20);
+
+    for (let idx = 0; idx < limited.length; idx++) {
+      try {
+        const loc = limited[idx];
+        const src = (await loc.getAttribute('src')) ?? '';
+        const sandboxValue = await loc.getAttribute('sandbox');
+        const sandboxed = sandboxValue !== null;
+
+        let crossOrigin = false;
+        if (src) {
+          try {
+            const iframeHost = new URL(src, seedUrl).hostname;
+            crossOrigin = iframeHost !== seedHost;
+          } catch {
+            crossOrigin = false;
+          }
+        }
+
+        let accessible = false;
+        let contentSample = '';
+        let hasForm = false;
+
+        try {
+          const fl = page.frameLocator('iframe').nth(idx);
+          const body = await fl.locator('body').textContent({ timeout: 2000 });
+          if (body !== null) {
+            accessible = true;
+            contentSample = body.trim().slice(0, 300);
+          }
+          const formCount = await fl.locator('form, input, textarea, select').count();
+          hasForm = formCount > 0;
+        } catch {
+          // cross-origin or sandboxed — not accessible from parent frame
+        }
+
+        records.push({
+          src,
+          crossOrigin,
+          sandboxed,
+          sandboxValue,
+          accessible,
+          contentSample,
+          hasForm,
+        });
+      } catch {
+        // skip individual iframe errors
+      }
+    }
+  } catch {
+    // skip entirely if locator fails
+  }
+  return records;
+}
+
+async function detectPopups(page: Page, result: MultiContextResult): Promise<void> {
+  try {
+    const candidates = [
+      ...await page.locator('[onclick*="window.open"]').all(),
+      ...await page.locator('button:has-text("Open in")').all(),
+    ];
+    const limited = candidates.slice(0, 3);
+
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? '';
+        const outcome = await Promise.all([
+          page.waitForEvent('popup', { timeout: 2000 }),
+          trigger.click(),
+        ]).catch(() => null);
+
+        if (!outcome) continue;
+        const popup = outcome[0] as Page;
+        if (!popup) continue;
+
+        const url = popup.url();
+        let isOAuthLike = false;
+        try {
+          const host = new URL(url).hostname;
+          isOAuthLike = /google|github|microsoft|okta|auth0/.test(host);
+        } catch {
+          // not a valid URL
+        }
+
+        result.popups.push({ url, triggerLabel, isOAuthLike });
+        await popup.close().catch(() => {});
+      } catch {
+        // skip this trigger
+      }
+    }
+  } catch {
+    // skip popup detection entirely
+  }
+}
+
+async function detectDownloads(page: Page, result: MultiContextResult): Promise<void> {
+  try {
+    const candidates = [
+      ...await page.locator('a[download]').all(),
+      ...await page.locator('button:has-text("Export")').all(),
+      ...await page.locator('button:has-text("Download")').all(),
+    ];
+    const limited = candidates.slice(0, 3);
+
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? '';
+        const outcome = await Promise.all([
+          page.waitForEvent('download', { timeout: 2000 }),
+          trigger.click(),
+        ]).catch(() => null);
+
+        if (!outcome) continue;
+        const download = outcome[0] as unknown as import('playwright').Download;
+        if (!download) continue;
+
+        const filename = download.suggestedFilename();
+        const downloadUrl = download.url();
+        await download.cancel().catch(() => {});
+
+        result.downloads.push({ filename, downloadUrl, triggerLabel });
+      } catch {
+        // skip this trigger
+      }
+    }
+  } catch {
+    // skip download detection entirely
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exported functions
+// ---------------------------------------------------------------------------
+
+/**
+ * Attaches a listener to the BrowserContext that records every new tab
+ * opened while the listener is active. Returns an unsubscribe function.
+ *
+ * Call this before navigating / clicking so that tabs opened during the
+ * crawl are captured, then call the returned function to stop listening.
+ */
+export function attachNewTabListener(
+  context: BrowserContext,
+  seedUrl: string,
+  result: MultiContextResult,
+): () => void {
+  let seedHost = '';
+  try {
+    seedHost = new URL(seedUrl).hostname;
+  } catch {
+    // leave empty — all tabs will be treated as external
+  }
+
+  const handler = async (newPage: Page) => {
+    try {
+      await newPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});
+      const url = newPage.url();
+
+      if (!url || url === 'about:blank') {
+        await newPage.close().catch(() => {});
+        return;
+      }
+
+      let isInternal = false;
+      let httpStatus: number | undefined;
+
+      try {
+        const tabHost = new URL(url).hostname;
+        isInternal = tabHost === seedHost;
+      } catch {
+        // non-parseable URL — treat as external
+      }
+
+      if (!isInternal) {
+        try {
+          const controller = new AbortController();
+          const timer = setTimeout(() => controller.abort(), 3000);
+          const resp = await fetch(url, { method: 'HEAD', signal: controller.signal });
+          clearTimeout(timer);
+          httpStatus = resp.status;
+        } catch {
+          // network unreachable or CORS — leave httpStatus undefined
+        }
+      }
+
+      // linkText is not available at context level; caller fills it if needed
+      result.newTabs.push({ url, linkText: '', isInternal, httpStatus });
+      await newPage.close().catch(() => {});
+    } catch {
+      // swallow handler errors so the listener stays alive
+    }
+  };
+
+  context.on('page', handler);
+  return () => context.off('page', handler);
+}
+
+/**
+ * Probes the current page for multi-context browser events: iframes,
+ * popup windows, and file downloads.
+ *
+ * New-tab tracking is intentionally excluded here — it must be set up at
+ * the crawler level (before navigation) using `attachNewTabListener`.
+ */
+export async function probeMultiContext(
+  page: Page,
+  context: BrowserContext,
+  seedUrl: string,
+): Promise<MultiContextResult> {
+  void context; // reserved for future use (e.g. routing overrides)
+
+  const result: MultiContextResult = {
+    newTabs: [],
+    iframes: [],
+    popups: [],
+    downloads: [],
+  };
+
+  try {
+    result.iframes = await enumerateIframes(page, seedUrl);
+  } catch {
+    // keep empty
+  }
+
+  try {
+    await detectPopups(page, result);
+  } catch {
+    // keep empty
+  }
+
+  try {
+    await detectDownloads(page, result);
+  } catch {
+    // keep empty
+  }
+
+  return result;
+}