@detiq/crawler 1.1.13 → 1.1.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.13",
+  "version": "1.1.14",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -75,6 +75,16 @@ import { generateOpenApiSpec, specToYaml } from './openapi-generator.js';
 import { WebhookNotifier } from './webhook-notifier.js';
 import { generatePlaywrightTest } from './playwright-recorder.js';
 
+async function getCrawlTuning() {
+  const { getSystemConfig } = await import('@detiq/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return {
+    navTimeoutMs:     parseInt(String(cfg?.navTimeoutMs     ?? '15000'), 10),
+    defaultTimeoutMs: parseInt(String(cfg?.defaultTimeoutMs ?? '2000'),  10),
+    userAgent:        String(cfg?.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)'),
+  };
+}
+
 // Module-level cache for per-crawl LLM config — avoids redundant DB round-trips inside per-page loops.
 // Keyed by tenantId so multi-tenant workers don't share configs across tenants.
 const _crawlLLMCache = new Map<string, any>();
@@ -517,6 +527,8 @@ export async function crawlProject(opts: {
   /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
   onUrlsDiscovered?: (urls: string[]) => Promise<void>;
 }, onProgress?: (pct: number, step: string) => Promise<void>) {
+  const tuning = await getCrawlTuning();
+
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
 
   // Upload screenshot to cloud storage if configured, returns cloudUrl or undefined
@@ -860,8 +872,8 @@ export async function crawlProject(opts: {
   // Use resolvePage() to get the active V3Page (which is a full Playwright proxy).
   const page = await (stagehand as any).resolvePage();
   // Cap all Playwright operations so one stuck page can't freeze the crawl.
-  try { page.setDefaultTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_DEFAULT_TIMEOUT_MS ?? '') || 45_000); } catch { /* proxy may not expose this */ }
-  try { page.setDefaultNavigationTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_NAV_TIMEOUT_MS ?? '') || 30_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultTimeout(tuning.defaultTimeoutMs || 45_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultNavigationTimeout(tuning.navTimeoutMs || 30_000); } catch { /* proxy may not expose this */ }
   // Stagehand V3Page proxy only supports the 'console' event (page.on throws for others).
   // rawPage will be wired to a real Playwright Page via the CDP connection established below.
   // Declared as `let` so the network-observer block can populate it.
@@ -1681,7 +1693,7 @@ export async function crawlProject(opts: {
         visited.set(canonUrl, '_pending_');
 
         try {
-          const navErr = await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null as null, (e: any) => e);
+          const navErr = await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).then(() => null as null, (e: any) => e);
           if (navErr && /timeout/i.test(String((navErr as any)?.message ?? navErr))) {
             await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
           }
@@ -1755,7 +1767,7 @@ export async function crawlProject(opts: {
       for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
         const wp = await pwCtx.newPage();
         await wp.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }).catch(() => {});
-        wp.setDefaultTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_DEFAULT_TIMEOUT_MS ?? '') || 45_000);
+        wp.setDefaultTimeout(tuning.defaultTimeoutMs || 45_000);
         liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {})));
       }
     }
@@ -1890,7 +1902,7 @@ export async function crawlProject(opts: {
           }
 
           // Try fast 15 s first; only escalate to 30 s on timeout so happy-path pages don't wait
-          const navErr15 = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null, (e: any) => e);
+          const navErr15 = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).then(() => null, (e: any) => e);
           if (navErr15) {
             if (/timeout/i.test(String(navErr15?.message ?? navErr15))) {
               await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
@@ -2345,7 +2357,7 @@ export async function crawlProject(opts: {
 
           // 2. Active nav item — SPAs set aria-current="page" on the active route link
           try {
-            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: 2000 }).catch(() => null);
+            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
             if (activeNav) {
               const t = activeNav.trim();
               if (t.length > 1 && t.length < 60 && t.toLowerCase() !== brandName) return t;
@@ -2359,7 +2371,7 @@ export async function crawlProject(opts: {
               'main h2', '[role="main"] h2', '.page-header h2',
             ];
             for (const sel of selectors) {
-              const h = await page.locator(sel).first().textContent({ timeout: 2000 }).catch(() => null);
+              const h = await page.locator(sel).first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
               if (!h) continue;
               const t = h.trim();
               if (t.length < 2 || t.length > 80) continue;
@@ -5439,6 +5451,7 @@ async function runAuthDiscoveryPhase(
   origin: string,
   onProgress?: AuthDiscoveryProgress,
 ): Promise<{ captured: number; discoveredUrls: string[] }> {
+  const tuning = await getCrawlTuning();
   const queue: Array<{ url: string; source: AuthDiscoverySource }> = [];
   const queued = new Set<string>();
   const capturedKeys = new Set<string>();
@@ -5478,7 +5491,7 @@ async function runAuthDiscoveryPhase(
   while (queue.length > 0 && captured < AUTH_DISCOVERY_MAX_SCREENS) {
     const current = queue.shift()!;
     try {
-      const response = await page.goto(current.url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => null);
+      const response = await page.goto(current.url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).catch(() => null);
       if (response && response.status && response.status() >= 400) continue;
       await page.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => { });
       await waitForStableDOM(page);

package/src/http-engine.ts

@@ -70,6 +70,18 @@ const SKIP_PATH = _skipPathExtra.length > 0
   ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')
   : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
 
+async function getHttpConfig() {
+  const { getSystemConfig } = await import('@detiq/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return {
+    retryLimit:   parseInt(String(cfg?.httpRetryLimit    ?? '5'),     10),
+    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs  ?? '60000'), 10),
+    retry5xx:     parseInt(String(cfg?.http5xxRetryLimit ?? '3'),     10),
+    timeoutMs:    parseInt(String(cfg?.httpTimeoutMs     ?? '15000'), 10),
+    concurrency:  parseInt(String(cfg?.httpConcurrency   ?? '5'),     10),
+  };
+}
+
 export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
 
 export interface HttpScreen {
@@ -101,7 +113,7 @@ export interface HttpCrawlOpts {
   urlFilter?: string;             // regex pattern — only crawl matching URLs
   traversalStrategy?: 'BFS' | 'DFS'; // default BFS
   outputFormats?: HttpOutputFormat[];
-  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'
+  cacheDir?: string;              // default '/tmp/zeta-cache'
   userAgent?: string;
   headers?: Record<string, string>;
   respectRobotsTxt?: boolean;     // default true
@@ -200,8 +212,10 @@ async function fetchWithBackoff(
   url: string,
   cache: ResponseCache,
   opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },
-  retryCount = 0
+  retryCount = 0,
+  httpCfg?: Awaited<ReturnType<typeof getHttpConfig>>
 ): Promise<FetchResult> {
+  if (!httpCfg) httpCfg = await getHttpConfig();
   const t0 = performance.now();
 
   if (cache.isFresh(url)) {
@@ -255,20 +269,20 @@ async function fetchWithBackoff(
 
   // 429 / 503 — exponential backoff with jitter
   if (response.status === 429 || response.status === 503) {
-    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);
     const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
-    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));
+    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(httpCfg.backoffCapMs, 1_000 * Math.pow(2, retryCount));
     const jitter = Math.random() * 1000;
-    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);
     await new Promise(r => setTimeout(r, baseDelay + jitter));
-    return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
   }
 
   // 5xx — retry up to 3 times
   if (response.status >= 500) {
-    if (retryCount < 3) {
+    if (retryCount < httpCfg.retry5xx) {
       await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
-      return fetchWithBackoff(url, cache, opts, retryCount + 1);
+      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
     }
     if (cache.staleIfError(url)) {
       const cached = cache.get(url)!;
@@ -292,11 +306,11 @@ async function fetchWithBackoff(
 
 export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
   const startTime = performance.now();
-  const maxScreens = opts.maxScreens ?? (parseInt(process.env.CRAWLER_HTTP_ENGINE_MAX_SCREENS ?? '') || 100);
-  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
-  const timeoutMs = opts.timeoutMs ?? 15_000;
-  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
-  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
+  const httpCfg = await getHttpConfig();
+  const maxScreens = opts.maxScreens ?? 100;
+  const cacheDir = opts.cacheDir ?? '/tmp/zeta-cache';
+  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;
+  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
   const cache = new ResponseCache(cacheDir);
   const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
   const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;
@@ -340,7 +354,7 @@ export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
 
       let fetchResult: FetchResult;
       try {
-        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);
         if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
       } catch (err: any) {
         console.error(`[http-engine] error fetching ${url}: ${err.message}`);
@@ -399,7 +413,7 @@ export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
     }));
   };
 
-  const concurrency = opts.concurrency ?? 5;
+  const concurrency = opts.concurrency ?? httpCfg.concurrency;
 
   while (queue.length > 0 && screens.length < maxScreens) {
     const batch = queue.splice(0, concurrency);

package/src/job-queue.ts

@@ -1703,7 +1703,9 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
     const appUrl: string = config.appUrl ?? crawlCfg.appUrl;
     if (!appUrl) throw new Error('No app URL configured. Save the crawl settings first.');
 
-    const maxDiscoverUrls: number = config.maxDiscoverUrls ?? 2000;
+    const { getSystemConfig } = await import('@detiq/core');
+    const crawlTuning = await getSystemConfig('crawl_tuning');
+    const maxDiscoverUrls: number = config.maxDiscoverUrls ?? parseInt(String(crawlTuning?.maxDiscoverUrls ?? '2000'), 10);
     const proxyUrl: string | undefined = config.proxyUrl ?? crawlCfg.proxyUrl;
     const screenshotDir = os.tmpdir();
     const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);

package/src/sitemap.ts

@@ -13,7 +13,12 @@ import { logger } from './logger.js';
 const FETCH_TIMEOUT_MS = 10_000;
 const MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)
 const MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index
-const MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory
+
+async function getSitemapMaxUrls(): Promise<number> {
+  const { getSystemConfig } = await import('@detiq/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return parseInt(String(cfg?.sitemapMaxUrls ?? '5000'), 10);
+}
 
 function toArray<T>(x: T | T[] | undefined | null): T[] {
   if (x == null) return [];
@@ -44,7 +49,7 @@ export function sameSite(candidateUrl: string, siteOrigin: string): boolean {
  * and deduped. Pure function — no I/O — for direct unit testing without a
  * real HTTP fetch.
  */
-export function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {
+export function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls = 5000): { pageUrls: string[]; subSitemapUrls: string[] } {
   const rawPageUrls: string[] = [];
   let subSitemapUrls: string[] = [];
 
@@ -66,7 +71,7 @@ export function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pag
   // Same-site only (www/protocol-tolerant) — a sitemap listing external URLs shouldn't seed this project's queue.
   const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
 
-  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };
 }
 
 async function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {
@@ -103,22 +108,23 @@ async function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | nu
  * Fetches and parses a sitemap (or sitemap index, recursed up to a small
  * depth) into a flat, deduped list of same-origin URLs.
  */
-export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {
+export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any, maxUrls?: number): Promise<string[]> {
   if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  if (maxUrls === undefined) maxUrls = await getSitemapMaxUrls();
   const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
   if (!parsed) return [];
 
-  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);
   const collected = [...pageUrls];
 
-  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {
     const subResults = await Promise.all(
-      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))
     );
     for (const sub of subResults) collected.push(...sub);
   }
 
-  return [...new Set(collected)].slice(0, MAX_URLS);
+  return [...new Set(collected)].slice(0, maxUrls);
 }
 
 /**
@@ -192,6 +198,7 @@ export async function discoverProjectUrls(
   try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }
 
   const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+  const maxUrls = await getSitemapMaxUrls();
 
   // 1. robots.txt → declared sitemaps
   let robotsSitemaps: string[] = [];
@@ -216,11 +223,11 @@ export async function discoverProjectUrls(
   const seen = new Map<string, UrlCandidate>();
   let sitemapFound = false;
   for (const sm of sitemapTargets) {
-    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);
     if (urls.length) sitemapFound = true;
     const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';
     for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
   }
 
-  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
+  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };
 }