@detiq/crawler 1.1.11 → 1.1.13

package/src/crawler.ts

@@ -155,9 +155,9 @@ type AuthDiscoveryOptions = {
   storageConfig?: any;
 };
 
-const AUTH_DISCOVERY_MAX_SCREENS = 36;
-const AUTH_DISCOVERY_MAX_QUEUE = 48;
-const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = 6;
+const AUTH_DISCOVERY_MAX_SCREENS = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_SCREENS ?? '') || 36;
+const AUTH_DISCOVERY_MAX_QUEUE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_QUEUE ?? '') || 48;
+const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE ?? '') || 6;
 const AUTH_DISCOVERY_PATHS = [
   '/',
   '/login', '/log-in', '/signin', '/sign-in', '/auth', '/auth/login', '/account/login', '/user/login', '/app/login',
@@ -576,15 +576,15 @@ export async function crawlProject(opts: {
             break;
           case 'openrouter':
             stagehandModel = `openai/${m}`;
-            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || 'https://openrouter.ai/api/v1' };
+            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_OPENROUTER_BASE_URL || 'https://openrouter.ai/api/v1' };
             break;
           case 'ollama':
             stagehandModel = `ollama/${m}`;
-            stagehandModelOpts = { baseURL: baseUrl || 'http://localhost:11434' };
+            stagehandModelOpts = { baseURL: baseUrl || process.env.CRAWLER_OLLAMA_BASE_URL || 'http://localhost:11434' };
             break;
           case 'nvidia':
             stagehandModel = `openai/${m.startsWith('openai/') ? m.slice(7) : m}`;
-            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || 'https://integrate.api.nvidia.com/v1' };
+            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_NVIDIA_BASE_URL || 'https://integrate.api.nvidia.com/v1' };
             break;
           case 'groq':
             stagehandModel = `groq/${m}`;
@@ -635,15 +635,18 @@ export async function crawlProject(opts: {
     const preferAnthropic = envModel.includes('claude') || (!envModel && !process.env.OPENAI_API_KEY);
     if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+      const defaultAnthropicModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel}`;
       stagehandConfigured = true;
     } else if (process.env.OPENAI_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.OPENAI_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || 'gpt-4o'}`;
+      const defaultOpenaiModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'gpt-4o';
+      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || defaultOpenaiModel}`;
       stagehandConfigured = true;
     } else if (process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+      const defaultAnthropicModel2 = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel2}`;
       stagehandConfigured = true;
     }
   }
@@ -848,7 +851,7 @@ export async function crawlProject(opts: {
     verbose: 1,
     logger: stagehandLogger,
     localBrowserLaunchOptions: { cdpUrl, headless: true },
-    domSettleTimeout: 2000,
+    domSettleTimeout: parseInt(process.env.CRAWLER_DOM_SETTLE_TIMEOUT_MS ?? '') || 2000,
   });
 
   await stagehand.init();
@@ -857,8 +860,8 @@ export async function crawlProject(opts: {
   // Use resolvePage() to get the active V3Page (which is a full Playwright proxy).
   const page = await (stagehand as any).resolvePage();
   // Cap all Playwright operations so one stuck page can't freeze the crawl.
-  try { page.setDefaultTimeout(45_000); } catch { /* proxy may not expose this */ }
-  try { page.setDefaultNavigationTimeout(30_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_DEFAULT_TIMEOUT_MS ?? '') || 45_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultNavigationTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_NAV_TIMEOUT_MS ?? '') || 30_000); } catch { /* proxy may not expose this */ }
   // Stagehand V3Page proxy only supports the 'console' event (page.on throws for others).
   // rawPage will be wired to a real Playwright Page via the CDP connection established below.
   // Declared as `let` so the network-observer block can populate it.
@@ -1472,7 +1475,7 @@ export async function crawlProject(opts: {
         ];
     const queued = new Set<string>(seedUrls.map(canonicalize));
     const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter);
-    const deduplicator = new ContentDeduplicator(0.88);
+    const deduplicator = new ContentDeduplicator(parseFloat(process.env.CRAWLER_DEDUP_THRESHOLD ?? '') || 0.88);
     const deadLinkTracker = new DeadLinkTracker();
     const redirectRegistry = new RedirectRegistry();
     // Re-seed from previously-discovered screens so deep routes found by AI link
@@ -1626,7 +1629,12 @@ export async function crawlProject(opts: {
     // Legal/policy-only pages — no interactive elements worth testing; skip to prevent noise
     // NOTE: /about, /contact, /careers, /help are intentionally NOT skipped — they have
     // real test targets (contact forms, job apply forms, help search, team content)
-    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
+    const skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
+      ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
+      : [];
+    const SKIP_PATH = skipPathExtra.length > 0
+      ? new RegExp(`/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap|${skipPathExtra.join('|')})([-/]|$)`, 'i')
+      : /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
     const segIdx = opts.segmentIndex ?? 1;
 
     const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
@@ -1637,6 +1645,7 @@ export async function crawlProject(opts: {
     // (was: 3 errors → skip 10 screens — too aggressive for Nvidia NIM which rate-limits quickly)
     let aiErrorStreak = 0;
     let aiSkipCount = 0;
+    const aiMinElements = parseInt(process.env.CRAWL_AI_LINK_MIN_ELEMENTS ?? '') || 5;
     let aiRateLimitWarned = false;
     // Hard guard: total URLs attempted (including skips) capped at maxScreens * 4
     // to prevent infinite-URL-explosion from WooCommerce/Shopify facet filter combos
@@ -1746,7 +1755,7 @@ export async function crawlProject(opts: {
       for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
         const wp = await pwCtx.newPage();
         await wp.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }).catch(() => {});
-        wp.setDefaultTimeout(45_000);
+        wp.setDefaultTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_DEFAULT_TIMEOUT_MS ?? '') || 45_000);
         liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {})));
       }
     }
@@ -2469,22 +2478,68 @@ export async function crawlProject(opts: {
         // FM-4: capture device pixel ratio for coordinate scaling in element highlighting
         const pageDpr: number = await page.evaluate(() => window.devicePixelRatio ?? 1).catch(() => 1);
 
-        // Parallelize: screenshot + element extraction + link discovery + AT capture (all work on current page state)
-        const [, elements, nextLinks, atElements] = await Promise.all([
+        // Extract elements first so we can highlight them in the screenshot
+        const elements = await extractElements(stagehand, page, elementLLM ?? undefined).catch((err: any) => {
+          console.error('[crawler] extractElements failed:', err);
+          return [] as Awaited<ReturnType<typeof extractElements>>;
+        });
+
+        // Inject bounding boxes for the screenshot
+        await page.evaluate((elems: any[]) => {
+          const container = document.createElement('div');
+          container.id = '__zeta_element_highlights';
+          container.style.position = 'absolute';
+          container.style.top = '0';
+          container.style.left = '0';
+          container.style.width = '100%';
+          container.style.height = '100%';
+          container.style.pointerEvents = 'none';
+          container.style.zIndex = '2147483647';
+          
+          elems.forEach((el, index) => {
+            if (!el.boundingRect) return;
+            const rect = el.boundingRect;
+            const box = document.createElement('div');
+            box.style.position = 'absolute';
+            box.style.left = (rect.x + window.scrollX) + 'px';
+            box.style.top = (rect.y + window.scrollY) + 'px';
+            box.style.width = rect.width + 'px';
+            box.style.height = rect.height + 'px';
+            box.style.border = '2px solid rgba(255, 0, 0, 0.8)';
+            box.style.backgroundColor = 'rgba(255, 0, 0, 0.1)';
+            box.style.boxSizing = 'border-box';
+            
+            const label = document.createElement('div');
+            label.textContent = el.role || 'element';
+            label.style.position = 'absolute';
+            label.style.top = '-16px';
+            label.style.left = '-2px';
+            label.style.backgroundColor = 'rgba(255, 0, 0, 0.9)';
+            label.style.color = '#FFF';
+            label.style.fontSize = '10px';
+            label.style.padding = '2px 4px';
+            label.style.borderRadius = '2px';
+            label.style.whiteSpace = 'nowrap';
+            label.style.fontFamily = 'monospace';
+            box.appendChild(label);
+            
+            container.appendChild(box);
+          });
+          document.body.appendChild(container);
+        }, elements).catch(() => {});
+
+        // Parallelize: screenshot + link discovery + AT capture
+        const [, nextLinks, atElements] = await Promise.all([
           page.screenshot({ path: shotPath, fullPage: true })
             .then(async () => {
               const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
               return (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl, pageDpr);
             })
             .catch((err: any) => console.error('[crawler] page.screenshot failed:', err)),
-          extractElements(stagehand, page, elementLLM ?? undefined).catch((err: any) => {
-            console.error('[crawler] extractElements failed:', err);
-            return [] as Awaited<ReturnType<typeof extractElements>>;
-          }),
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
           // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
-          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < 5)
+          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < aiMinElements)
             .then((freshLinks) => {
               if (aiSkipCount > 0) aiSkipCount--;
               aiErrorStreak = 0;
@@ -2517,6 +2572,12 @@ export async function crawlProject(opts: {
             }),
           readScreen(page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>),
         ]);
+
+        // Cleanup highlights
+        await page.evaluate(() => {
+          const container = document.getElementById('__zeta_element_highlights');
+          if (container) container.remove();
+        }).catch(() => {});
         logger.info({ url, screenId: screen.id, version: screen.version, elements: elements.length, links: nextLinks.length, atSample: atElements.slice(0, 3).map((a: any) => a.label) }, '[crawler] screen captured');
         console.log('[crawler] AT:', JSON.stringify(atElements.slice(0, 5)));
         // Fire-and-forget keyboard-nav probe — records Tab sequence and focus-indicator issues per screen
@@ -2912,12 +2973,15 @@ export async function crawlProject(opts: {
           // P3a: Post-extraction DOM count — detect async-loaded content.
           // Only check sparse pages (preDomCount < 25) where async widgets are likely.
           // If DOM grew significantly during AI extraction, re-visit later with networkidle.
-          if (preDomCount < 25 && !revisitQueue.includes(url)) {
+          const asyncContentCheckMinDom = parseInt(process.env.CRAWLER_ASYNC_CONTENT_MIN_DOM ?? '') || 25;
+          if (preDomCount < asyncContentCheckMinDom && !revisitQueue.includes(url)) {
             try {
               const postDomCount = await page.evaluate(() =>
                 document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
               ).catch(() => 0);
-              if (postDomCount > preDomCount * 1.3 + 4) {
+              const asyncGrowthRatio = parseFloat(process.env.CRAWLER_ASYNC_GROWTH_RATIO ?? '') || 1.3;
+              const asyncGrowthFloor = parseInt(process.env.CRAWLER_ASYNC_GROWTH_FLOOR ?? '') || 4;
+              if (postDomCount > preDomCount * asyncGrowthRatio + asyncGrowthFloor) {
                 revisitQueue.push(url);
                 logger.info({ url, preDomCount, postDomCount }, '[P3a] Async content detected — scheduled for re-visit');
               }
@@ -2926,7 +2990,7 @@ export async function crawlProject(opts: {
 
           // P2: Static JS router detection — scan bundle for declared client-side routes.
           // Only run on first 5 pages (bundle is the same across pages; scanning every page is wasteful).
-          if (visited.size <= 5) {
+          if (visited.size <= (parseInt(process.env.CRAWLER_SPA_ROUTER_DETECT_LIMIT ?? '') || 5)) {
             try {
               const jsDetectedRoutes = await detectJsRoutes(page, origin);
               for (const route of jsDetectedRoutes) {
@@ -2959,7 +3023,7 @@ export async function crawlProject(opts: {
           // Triggered per-page when: zero DOM links found + fewer than 5 elements (dead-end / canvas page).
           // Skyvern returns URLs it discovered → added to queue for ZeTa's own BFS crawl (not saved directly).
           // ZeTa does all actual crawling — Skyvern is only a URL-discovery override layer.
-          if (nextLinks.length === 0 && elements.length < 5 && opts.skyvernConfig) {
+          if (nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? '') || 5) && opts.skyvernConfig) {
             try {
               logger.info({ url }, 'Gap 5: thin page detected — asking Skyvern to discover navigation URLs');
               const skyvernNav = await skyvernNavigate(
@@ -3073,7 +3137,8 @@ export async function crawlProject(opts: {
     // Skyvern is ONLY a URL-discovery layer here — it does NOT save screens directly.
     // Use only successfully-saved screens (screenId present), not total discovered including errors.
     const successfulScreenCount = discovered.filter(d => d.screenId).length;
-    if (successfulScreenCount < 10 && opts.skyvernConfig) {
+    const skyvernMinScreens = parseInt(process.env.CRAWL_SKYVERN_MIN_SCREENS ?? '') || 10;
+    if (successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
       await onProgress?.(78, 'Few screens found — using Skyvern to discover additional navigation URLs');
       const currentUrl = page.url();
       const skyvernNav = await skyvernNavigate(
@@ -3231,7 +3296,7 @@ export async function crawlProject(opts: {
     // P6: Gap Pass — LLM gap analysis on discovered screens to find structurally missing pages.
     // Runs after P3a re-visits so the gap analysis has the most complete screen list possible.
     // Confirmed gap URLs (HEAD=200) are visited directly with the existing browser.
-    if (visited.size >= 3) {
+    if (visited.size >= (parseInt(process.env.CRAWLER_GAP_PASS_MIN_SCREENS ?? '') || 3)) {
       try {
         await onProgress?.(80, 'Running gap analysis…');
         const llmCfgForGap = await getCachedCrawlLLM(opts.tenantId);
@@ -3257,7 +3322,7 @@ export async function crawlProject(opts: {
 
           // Visit each confirmed gap URL with the existing browser (depth=1, no link traversal)
           const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
-          for (const gap of confirmedGaps.slice(0, 8)) {
+          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? '') || 8)) {
             const gapUrl = gap.confirmedUrl!;
             if (visited.size >= maxScreens) break;
             try {
@@ -3852,7 +3917,7 @@ async function discoverLinks(stagehand: Stagehand, origin: string, page: any, ll
   }
 }
 
-async function domFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
+export async function domFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
   return page.evaluate(() => {
     const out: { meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }[] = [];
     const seen = new Set<string>();

package/src/frontier.ts

@@ -27,11 +27,13 @@ function scoreUrl(url: string, keywords: string[]): number {
   try {
     const u = new URL(url);
     const segments = u.pathname.split('/').filter(Boolean);
-    let score = -segments.length;
+    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? '') || 1;
+    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? '') || 10;
+    let score = -segments.length * depthPenaltyWeight;
     if (keywords.length > 0) {
       const lower = url.toLowerCase();
       for (const kw of keywords) {
-        if (kw && lower.includes(kw)) score += 10;
+        if (kw && lower.includes(kw)) score += keywordBoostWeight;
       }
     }
     return score;
@@ -69,7 +71,7 @@ export class CrawlFrontier {
     const sparseBoost =
       this.strategy === 'BEST_FIRST' &&
       pushOpts?.parentElements !== undefined &&
-      pushOpts.parentElements < 5
+      pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? '') || 5)
         ? 1
         : 0;
     this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });

package/src/gap-pass.ts

@@ -73,7 +73,7 @@ Respond with ONLY valid JSON, no markdown:
     const Anthropic = (await import('@anthropic-ai/sdk')).default;
     const client = new Anthropic({ apiKey: llmCfg.apiKey });
     const response = await client.messages.create({
-      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+      model: llmCfg.model ?? process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
       temperature: 0,
       // @ts-ignore

package/src/http-engine.ts

@@ -63,7 +63,12 @@ async function getChromeFetchDispatcher() {
 const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
 
 // Pages unlikely to have testable UI — skip from HTTP crawl queue
-const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+const _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
+  ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const SKIP_PATH = _skipPathExtra.length > 0
+  ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')
+  : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
 
 export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
 
@@ -212,7 +217,7 @@ async function fetchWithBackoff(
     'Accept-Language': 'en-US,en;q=0.9',
     'Accept-Encoding': 'gzip, deflate, br, zstd',
     // Chrome-like security headers reduce bot-detection fingerprint
-    'sec-ch-ua': '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+    'sec-ch-ua': `"Google Chrome";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Chromium";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Not-A.Brand";v="99"`,
     'sec-ch-ua-mobile': '?0',
     'sec-ch-ua-platform': '"Windows"',
     'Sec-Fetch-Dest': 'document',
@@ -287,10 +292,10 @@ async function fetchWithBackoff(
 
 export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
   const startTime = performance.now();
-  const maxScreens = opts.maxScreens ?? 100;
+  const maxScreens = opts.maxScreens ?? (parseInt(process.env.CRAWLER_HTTP_ENGINE_MAX_SCREENS ?? '') || 100);
   const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
   const timeoutMs = opts.timeoutMs ?? 15_000;
-  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)';
+  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
   const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
   const cache = new ResponseCache(cacheDir);
   const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;