@detiq/crawler 1.1.12 → 1.1.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.12",
+  "version": "1.1.14",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/content-chunker.ts

@@ -21,9 +21,9 @@ export function estimateTokens(text: string): number {
 }
 
 export function chunkContent(text: string, opts: ChunkOpts): Chunk[] {
-  const size = opts.chunkSize ?? 2000;
+  const size = opts.chunkSize ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_SIZE ?? '') || 2000);
   const charSize = size * 4;
-  const overlap = opts.overlap ?? 200;
+  const overlap = opts.overlap ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_OVERLAP ?? '') || 200);
   const overlapChars = overlap * 4;
 
   switch (opts.strategy) {

package/src/crawler.ts

@@ -75,6 +75,16 @@ import { generateOpenApiSpec, specToYaml } from './openapi-generator.js';
 import { WebhookNotifier } from './webhook-notifier.js';
 import { generatePlaywrightTest } from './playwright-recorder.js';
 
+async function getCrawlTuning() {
+  const { getSystemConfig } = await import('@detiq/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return {
+    navTimeoutMs:     parseInt(String(cfg?.navTimeoutMs     ?? '15000'), 10),
+    defaultTimeoutMs: parseInt(String(cfg?.defaultTimeoutMs ?? '2000'),  10),
+    userAgent:        String(cfg?.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)'),
+  };
+}
+
 // Module-level cache for per-crawl LLM config — avoids redundant DB round-trips inside per-page loops.
 // Keyed by tenantId so multi-tenant workers don't share configs across tenants.
 const _crawlLLMCache = new Map<string, any>();
@@ -155,9 +165,9 @@ type AuthDiscoveryOptions = {
   storageConfig?: any;
 };
 
-const AUTH_DISCOVERY_MAX_SCREENS = 36;
-const AUTH_DISCOVERY_MAX_QUEUE = 48;
-const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = 6;
+const AUTH_DISCOVERY_MAX_SCREENS = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_SCREENS ?? '') || 36;
+const AUTH_DISCOVERY_MAX_QUEUE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_QUEUE ?? '') || 48;
+const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE ?? '') || 6;
 const AUTH_DISCOVERY_PATHS = [
   '/',
   '/login', '/log-in', '/signin', '/sign-in', '/auth', '/auth/login', '/account/login', '/user/login', '/app/login',
@@ -517,6 +527,8 @@ export async function crawlProject(opts: {
   /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
   onUrlsDiscovered?: (urls: string[]) => Promise<void>;
 }, onProgress?: (pct: number, step: string) => Promise<void>) {
+  const tuning = await getCrawlTuning();
+
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
 
   // Upload screenshot to cloud storage if configured, returns cloudUrl or undefined
@@ -576,15 +588,15 @@ export async function crawlProject(opts: {
             break;
           case 'openrouter':
             stagehandModel = `openai/${m}`;
-            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || 'https://openrouter.ai/api/v1' };
+            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_OPENROUTER_BASE_URL || 'https://openrouter.ai/api/v1' };
             break;
           case 'ollama':
             stagehandModel = `ollama/${m}`;
-            stagehandModelOpts = { baseURL: baseUrl || 'http://localhost:11434' };
+            stagehandModelOpts = { baseURL: baseUrl || process.env.CRAWLER_OLLAMA_BASE_URL || 'http://localhost:11434' };
             break;
           case 'nvidia':
             stagehandModel = `openai/${m.startsWith('openai/') ? m.slice(7) : m}`;
-            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || 'https://integrate.api.nvidia.com/v1' };
+            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_NVIDIA_BASE_URL || 'https://integrate.api.nvidia.com/v1' };
             break;
           case 'groq':
             stagehandModel = `groq/${m}`;
@@ -635,15 +647,18 @@ export async function crawlProject(opts: {
     const preferAnthropic = envModel.includes('claude') || (!envModel && !process.env.OPENAI_API_KEY);
     if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+      const defaultAnthropicModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel}`;
       stagehandConfigured = true;
     } else if (process.env.OPENAI_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.OPENAI_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || 'gpt-4o'}`;
+      const defaultOpenaiModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'gpt-4o';
+      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || defaultOpenaiModel}`;
       stagehandConfigured = true;
     } else if (process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+      const defaultAnthropicModel2 = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel2}`;
       stagehandConfigured = true;
     }
   }
@@ -848,7 +863,7 @@ export async function crawlProject(opts: {
     verbose: 1,
     logger: stagehandLogger,
     localBrowserLaunchOptions: { cdpUrl, headless: true },
-    domSettleTimeout: 2000,
+    domSettleTimeout: parseInt(process.env.CRAWLER_DOM_SETTLE_TIMEOUT_MS ?? '') || 2000,
   });
 
   await stagehand.init();
@@ -857,8 +872,8 @@ export async function crawlProject(opts: {
   // Use resolvePage() to get the active V3Page (which is a full Playwright proxy).
   const page = await (stagehand as any).resolvePage();
   // Cap all Playwright operations so one stuck page can't freeze the crawl.
-  try { page.setDefaultTimeout(45_000); } catch { /* proxy may not expose this */ }
-  try { page.setDefaultNavigationTimeout(30_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultTimeout(tuning.defaultTimeoutMs || 45_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultNavigationTimeout(tuning.navTimeoutMs || 30_000); } catch { /* proxy may not expose this */ }
   // Stagehand V3Page proxy only supports the 'console' event (page.on throws for others).
   // rawPage will be wired to a real Playwright Page via the CDP connection established below.
   // Declared as `let` so the network-observer block can populate it.
@@ -1472,7 +1487,7 @@ export async function crawlProject(opts: {
         ];
     const queued = new Set<string>(seedUrls.map(canonicalize));
     const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter);
-    const deduplicator = new ContentDeduplicator(0.88);
+    const deduplicator = new ContentDeduplicator(parseFloat(process.env.CRAWLER_DEDUP_THRESHOLD ?? '') || 0.88);
     const deadLinkTracker = new DeadLinkTracker();
     const redirectRegistry = new RedirectRegistry();
     // Re-seed from previously-discovered screens so deep routes found by AI link
@@ -1626,7 +1641,12 @@ export async function crawlProject(opts: {
     // Legal/policy-only pages — no interactive elements worth testing; skip to prevent noise
     // NOTE: /about, /contact, /careers, /help are intentionally NOT skipped — they have
     // real test targets (contact forms, job apply forms, help search, team content)
-    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
+    const skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
+      ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
+      : [];
+    const SKIP_PATH = skipPathExtra.length > 0
+      ? new RegExp(`/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap|${skipPathExtra.join('|')})([-/]|$)`, 'i')
+      : /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
     const segIdx = opts.segmentIndex ?? 1;
 
     const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
@@ -1637,6 +1657,7 @@ export async function crawlProject(opts: {
     // (was: 3 errors → skip 10 screens — too aggressive for Nvidia NIM which rate-limits quickly)
     let aiErrorStreak = 0;
     let aiSkipCount = 0;
+    const aiMinElements = parseInt(process.env.CRAWL_AI_LINK_MIN_ELEMENTS ?? '') || 5;
     let aiRateLimitWarned = false;
     // Hard guard: total URLs attempted (including skips) capped at maxScreens * 4
     // to prevent infinite-URL-explosion from WooCommerce/Shopify facet filter combos
@@ -1672,7 +1693,7 @@ export async function crawlProject(opts: {
         visited.set(canonUrl, '_pending_');
 
         try {
-          const navErr = await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null as null, (e: any) => e);
+          const navErr = await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).then(() => null as null, (e: any) => e);
           if (navErr && /timeout/i.test(String((navErr as any)?.message ?? navErr))) {
             await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
           }
@@ -1746,7 +1767,7 @@ export async function crawlProject(opts: {
       for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
         const wp = await pwCtx.newPage();
         await wp.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }).catch(() => {});
-        wp.setDefaultTimeout(45_000);
+        wp.setDefaultTimeout(tuning.defaultTimeoutMs || 45_000);
         liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {})));
       }
     }
@@ -1881,7 +1902,7 @@ export async function crawlProject(opts: {
           }
 
           // Try fast 15 s first; only escalate to 30 s on timeout so happy-path pages don't wait
-          const navErr15 = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null, (e: any) => e);
+          const navErr15 = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).then(() => null, (e: any) => e);
           if (navErr15) {
             if (/timeout/i.test(String(navErr15?.message ?? navErr15))) {
               await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
@@ -2336,7 +2357,7 @@ export async function crawlProject(opts: {
 
           // 2. Active nav item — SPAs set aria-current="page" on the active route link
           try {
-            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: 2000 }).catch(() => null);
+            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
             if (activeNav) {
               const t = activeNav.trim();
               if (t.length > 1 && t.length < 60 && t.toLowerCase() !== brandName) return t;
@@ -2350,7 +2371,7 @@ export async function crawlProject(opts: {
               'main h2', '[role="main"] h2', '.page-header h2',
             ];
             for (const sel of selectors) {
-              const h = await page.locator(sel).first().textContent({ timeout: 2000 }).catch(() => null);
+              const h = await page.locator(sel).first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
               if (!h) continue;
               const t = h.trim();
               if (t.length < 2 || t.length > 80) continue;
@@ -2530,7 +2551,7 @@ export async function crawlProject(opts: {
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
           // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
-          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < 5)
+          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < aiMinElements)
             .then((freshLinks) => {
               if (aiSkipCount > 0) aiSkipCount--;
               aiErrorStreak = 0;
@@ -2964,12 +2985,15 @@ export async function crawlProject(opts: {
           // P3a: Post-extraction DOM count — detect async-loaded content.
           // Only check sparse pages (preDomCount < 25) where async widgets are likely.
           // If DOM grew significantly during AI extraction, re-visit later with networkidle.
-          if (preDomCount < 25 && !revisitQueue.includes(url)) {
+          const asyncContentCheckMinDom = parseInt(process.env.CRAWLER_ASYNC_CONTENT_MIN_DOM ?? '') || 25;
+          if (preDomCount < asyncContentCheckMinDom && !revisitQueue.includes(url)) {
             try {
               const postDomCount = await page.evaluate(() =>
                 document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
               ).catch(() => 0);
-              if (postDomCount > preDomCount * 1.3 + 4) {
+              const asyncGrowthRatio = parseFloat(process.env.CRAWLER_ASYNC_GROWTH_RATIO ?? '') || 1.3;
+              const asyncGrowthFloor = parseInt(process.env.CRAWLER_ASYNC_GROWTH_FLOOR ?? '') || 4;
+              if (postDomCount > preDomCount * asyncGrowthRatio + asyncGrowthFloor) {
                 revisitQueue.push(url);
                 logger.info({ url, preDomCount, postDomCount }, '[P3a] Async content detected — scheduled for re-visit');
               }
@@ -2978,7 +3002,7 @@ export async function crawlProject(opts: {
 
           // P2: Static JS router detection — scan bundle for declared client-side routes.
           // Only run on first 5 pages (bundle is the same across pages; scanning every page is wasteful).
-          if (visited.size <= 5) {
+          if (visited.size <= (parseInt(process.env.CRAWLER_SPA_ROUTER_DETECT_LIMIT ?? '') || 5)) {
             try {
               const jsDetectedRoutes = await detectJsRoutes(page, origin);
               for (const route of jsDetectedRoutes) {
@@ -3011,7 +3035,7 @@ export async function crawlProject(opts: {
           // Triggered per-page when: zero DOM links found + fewer than 5 elements (dead-end / canvas page).
           // Skyvern returns URLs it discovered → added to queue for ZeTa's own BFS crawl (not saved directly).
           // ZeTa does all actual crawling — Skyvern is only a URL-discovery override layer.
-          if (nextLinks.length === 0 && elements.length < 5 && opts.skyvernConfig) {
+          if (nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? '') || 5) && opts.skyvernConfig) {
             try {
               logger.info({ url }, 'Gap 5: thin page detected — asking Skyvern to discover navigation URLs');
               const skyvernNav = await skyvernNavigate(
@@ -3125,7 +3149,8 @@ export async function crawlProject(opts: {
     // Skyvern is ONLY a URL-discovery layer here — it does NOT save screens directly.
     // Use only successfully-saved screens (screenId present), not total discovered including errors.
     const successfulScreenCount = discovered.filter(d => d.screenId).length;
-    if (successfulScreenCount < 10 && opts.skyvernConfig) {
+    const skyvernMinScreens = parseInt(process.env.CRAWL_SKYVERN_MIN_SCREENS ?? '') || 10;
+    if (successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
       await onProgress?.(78, 'Few screens found — using Skyvern to discover additional navigation URLs');
       const currentUrl = page.url();
       const skyvernNav = await skyvernNavigate(
@@ -3283,7 +3308,7 @@ export async function crawlProject(opts: {
     // P6: Gap Pass — LLM gap analysis on discovered screens to find structurally missing pages.
     // Runs after P3a re-visits so the gap analysis has the most complete screen list possible.
     // Confirmed gap URLs (HEAD=200) are visited directly with the existing browser.
-    if (visited.size >= 3) {
+    if (visited.size >= (parseInt(process.env.CRAWLER_GAP_PASS_MIN_SCREENS ?? '') || 3)) {
       try {
         await onProgress?.(80, 'Running gap analysis…');
         const llmCfgForGap = await getCachedCrawlLLM(opts.tenantId);
@@ -3309,7 +3334,7 @@ export async function crawlProject(opts: {
 
           // Visit each confirmed gap URL with the existing browser (depth=1, no link traversal)
           const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
-          for (const gap of confirmedGaps.slice(0, 8)) {
+          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? '') || 8)) {
             const gapUrl = gap.confirmedUrl!;
             if (visited.size >= maxScreens) break;
             try {
@@ -5426,6 +5451,7 @@ async function runAuthDiscoveryPhase(
   origin: string,
   onProgress?: AuthDiscoveryProgress,
 ): Promise<{ captured: number; discoveredUrls: string[] }> {
+  const tuning = await getCrawlTuning();
   const queue: Array<{ url: string; source: AuthDiscoverySource }> = [];
   const queued = new Set<string>();
   const capturedKeys = new Set<string>();
@@ -5465,7 +5491,7 @@ async function runAuthDiscoveryPhase(
   while (queue.length > 0 && captured < AUTH_DISCOVERY_MAX_SCREENS) {
     const current = queue.shift()!;
     try {
-      const response = await page.goto(current.url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => null);
+      const response = await page.goto(current.url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).catch(() => null);
       if (response && response.status && response.status() >= 400) continue;
       await page.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => { });
       await waitForStableDOM(page);

package/src/frontier.ts

@@ -27,11 +27,13 @@ function scoreUrl(url: string, keywords: string[]): number {
   try {
     const u = new URL(url);
     const segments = u.pathname.split('/').filter(Boolean);
-    let score = -segments.length;
+    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? '') || 1;
+    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? '') || 10;
+    let score = -segments.length * depthPenaltyWeight;
     if (keywords.length > 0) {
       const lower = url.toLowerCase();
       for (const kw of keywords) {
-        if (kw && lower.includes(kw)) score += 10;
+        if (kw && lower.includes(kw)) score += keywordBoostWeight;
       }
     }
     return score;
@@ -69,7 +71,7 @@ export class CrawlFrontier {
     const sparseBoost =
       this.strategy === 'BEST_FIRST' &&
       pushOpts?.parentElements !== undefined &&
-      pushOpts.parentElements < 5
+      pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? '') || 5)
         ? 1
         : 0;
     this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });

package/src/gap-pass.ts

@@ -73,7 +73,7 @@ Respond with ONLY valid JSON, no markdown:
     const Anthropic = (await import('@anthropic-ai/sdk')).default;
     const client = new Anthropic({ apiKey: llmCfg.apiKey });
     const response = await client.messages.create({
-      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+      model: llmCfg.model ?? process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
       temperature: 0,
       // @ts-ignore

package/src/http-engine.ts

@@ -63,7 +63,24 @@ async function getChromeFetchDispatcher() {
 const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
 
 // Pages unlikely to have testable UI — skip from HTTP crawl queue
-const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+const _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
+  ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const SKIP_PATH = _skipPathExtra.length > 0
+  ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')
+  : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+
+async function getHttpConfig() {
+  const { getSystemConfig } = await import('@detiq/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return {
+    retryLimit:   parseInt(String(cfg?.httpRetryLimit    ?? '5'),     10),
+    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs  ?? '60000'), 10),
+    retry5xx:     parseInt(String(cfg?.http5xxRetryLimit ?? '3'),     10),
+    timeoutMs:    parseInt(String(cfg?.httpTimeoutMs     ?? '15000'), 10),
+    concurrency:  parseInt(String(cfg?.httpConcurrency   ?? '5'),     10),
+  };
+}
 
 export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
 
@@ -96,7 +113,7 @@ export interface HttpCrawlOpts {
   urlFilter?: string;             // regex pattern — only crawl matching URLs
   traversalStrategy?: 'BFS' | 'DFS'; // default BFS
   outputFormats?: HttpOutputFormat[];
-  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'
+  cacheDir?: string;              // default '/tmp/zeta-cache'
   userAgent?: string;
   headers?: Record<string, string>;
   respectRobotsTxt?: boolean;     // default true
@@ -195,8 +212,10 @@ async function fetchWithBackoff(
   url: string,
   cache: ResponseCache,
   opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },
-  retryCount = 0
+  retryCount = 0,
+  httpCfg?: Awaited<ReturnType<typeof getHttpConfig>>
 ): Promise<FetchResult> {
+  if (!httpCfg) httpCfg = await getHttpConfig();
   const t0 = performance.now();
 
   if (cache.isFresh(url)) {
@@ -212,7 +231,7 @@ async function fetchWithBackoff(
     'Accept-Language': 'en-US,en;q=0.9',
     'Accept-Encoding': 'gzip, deflate, br, zstd',
     // Chrome-like security headers reduce bot-detection fingerprint
-    'sec-ch-ua': '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+    'sec-ch-ua': `"Google Chrome";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Chromium";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Not-A.Brand";v="99"`,
     'sec-ch-ua-mobile': '?0',
     'sec-ch-ua-platform': '"Windows"',
     'Sec-Fetch-Dest': 'document',
@@ -250,20 +269,20 @@ async function fetchWithBackoff(
 
   // 429 / 503 — exponential backoff with jitter
   if (response.status === 429 || response.status === 503) {
-    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);
     const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
-    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));
+    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(httpCfg.backoffCapMs, 1_000 * Math.pow(2, retryCount));
     const jitter = Math.random() * 1000;
-    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);
     await new Promise(r => setTimeout(r, baseDelay + jitter));
-    return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
   }
 
   // 5xx — retry up to 3 times
   if (response.status >= 500) {
-    if (retryCount < 3) {
+    if (retryCount < httpCfg.retry5xx) {
       await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
-      return fetchWithBackoff(url, cache, opts, retryCount + 1);
+      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
     }
     if (cache.staleIfError(url)) {
       const cached = cache.get(url)!;
@@ -287,11 +306,11 @@ async function fetchWithBackoff(
 
 export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
   const startTime = performance.now();
+  const httpCfg = await getHttpConfig();
   const maxScreens = opts.maxScreens ?? 100;
-  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
-  const timeoutMs = opts.timeoutMs ?? 15_000;
-  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)';
-  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
+  const cacheDir = opts.cacheDir ?? '/tmp/zeta-cache';
+  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;
+  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
   const cache = new ResponseCache(cacheDir);
   const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
   const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;
@@ -335,7 +354,7 @@ export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
 
       let fetchResult: FetchResult;
       try {
-        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);
         if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
       } catch (err: any) {
         console.error(`[http-engine] error fetching ${url}: ${err.message}`);
@@ -394,7 +413,7 @@ export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
     }));
   };
 
-  const concurrency = opts.concurrency ?? 5;
+  const concurrency = opts.concurrency ?? httpCfg.concurrency;
 
   while (queue.length > 0 && screens.length < maxScreens) {
     const batch = queue.splice(0, concurrency);

package/src/job-queue.ts

@@ -383,12 +383,18 @@ export async function createTrackedJob(
   return job.id;
 }
 
-/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors. */
+/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
+ * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
 export async function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void> {
+  const nextRetryAt = error ? new Date(Date.now() + 60_000) : undefined;
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: error ? 'FAILED' : 'DONE', finishedAt: new Date(), ...(error ? { error } : { progress: 100 }) },
+      data: {
+        status: error ? 'FAILED' : 'DONE',
+        finishedAt: new Date(),
+        ...(error ? { error, nextRetryAt } : { progress: 100 }),
+      },
     })
   ).catch(() => {});
 }
@@ -471,7 +477,7 @@ export async function cancelJob(tenantId: string, jobId: string): Promise<void>
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: 'CANCELLED', finishedAt: new Date() },
+      data: { status: 'CANCELLED', finishedAt: new Date(), heartbeatAt: null, workerId: null },
     })
   );
 }
@@ -637,7 +643,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     await opts?.onProgress?.(pct);
   };
 
-  const DEFAULT_MAX_SCREENS = 150;
+  const DEFAULT_MAX_SCREENS = parseInt(process.env.CRAWL_DEFAULT_MAX_SCREENS ?? '') || 150;
 
   try {
     // Per-project maxScreens + deviceProfiles + seedUrls + interactive probing + sitemap seeding + traversal strategy from crawlConfig override job config defaults.
@@ -1611,7 +1617,7 @@ export async function processDataRetentionPurge(): Promise<void> {
       orderBy: { dataRetentionDays: 'asc' },
       select: { dataRetentionDays: true },
     });
-    const retentionDays = setting?.dataRetentionDays ?? 365;
+    const retentionDays = setting?.dataRetentionDays ?? (parseInt(process.env.DEFAULT_DATA_RETENTION_DAYS ?? '') || 365);
     const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000);
 
     const result = await prismaAdmin.auditLog.deleteMany({
@@ -1697,7 +1703,9 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
     const appUrl: string = config.appUrl ?? crawlCfg.appUrl;
     if (!appUrl) throw new Error('No app URL configured. Save the crawl settings first.');
 
-    const maxDiscoverUrls: number = config.maxDiscoverUrls ?? 2000;
+    const { getSystemConfig } = await import('@detiq/core');
+    const crawlTuning = await getSystemConfig('crawl_tuning');
+    const maxDiscoverUrls: number = config.maxDiscoverUrls ?? parseInt(String(crawlTuning?.maxDiscoverUrls ?? '2000'), 10);
     const proxyUrl: string | undefined = config.proxyUrl ?? crawlCfg.proxyUrl;
     const screenshotDir = os.tmpdir();
     const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);

package/src/policies/crawl-safety.ts

@@ -6,7 +6,13 @@ export interface CrawlSafetyDecision {
   severity: 'info' | 'warn' | 'block';
 }
 
-const DESTRUCTIVE_ACTION = /\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop)\b/i;
+const _extraDestructive = process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS
+  ? process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const DESTRUCTIVE_ACTION = new RegExp(
+  `\\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop${_extraDestructive.length ? `|${_extraDestructive.join('|')}` : ''})\\b`,
+  'i',
+);
 const SECRET_LIKE = /\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\b/i;
 
 export function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision {

package/src/profiles/domain-profile.ts

@@ -114,7 +114,7 @@ const PROFILES: Record<Industry, Omit<DomainProfile, 'confidence' | 'evidence'>>
   },
 };
 
-const INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
+const BASE_INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
   { industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
   { industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
   { industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
@@ -125,6 +125,29 @@ const INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
   { industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },
 ];
 
+// Allow additional industry hint keywords via ENV without replacing the baseline.
+// Format: JSON array of { industry: string; terms: string[] } — terms are treated as case-insensitive regex.
+// Example: EXTRA_INDUSTRY_HINTS_JSON='[{"industry":"saas","terms":["platform","multitenant"]}]'
+const _extraHints: Array<{ industry: Industry; terms: RegExp[] }> = (() => {
+  const raw = process.env.EXTRA_INDUSTRY_HINTS_JSON;
+  if (!raw) return [];
+  try {
+    const parsed: Array<{ industry: string; terms: string[] }> = JSON.parse(raw);
+    return parsed.map(h => ({ industry: h.industry as Industry, terms: h.terms.map(t => new RegExp(t, 'i')) }));
+  } catch { return []; }
+})();
+
+const INDUSTRY_HINTS = [...BASE_INDUSTRY_HINTS, ..._extraHints];
+
+// Allow patching profile fields (riskyActions, piiRules, etc.) per industry via ENV.
+// Format: JSON object keyed by industry, value is partial DomainProfile fields to merge.
+// Example: DOMAIN_PROFILE_OVERRIDES_JSON='{"saas":{"riskyActions":["delete workspace","export all data"]}}'
+const _profileOverrides: Partial<Record<Industry, Partial<Omit<DomainProfile, 'confidence' | 'evidence'>>>> = (() => {
+  const raw = process.env.DOMAIN_PROFILE_OVERRIDES_JSON;
+  if (!raw) return {};
+  try { return JSON.parse(raw); } catch { return {}; }
+})();
+
 export function classifyDomainProfile(input: {
   appUrl: string;
   projectName?: string | null;

package/src/rate-limiter.ts

@@ -30,8 +30,8 @@ export class DomainRateLimiter {
   private readonly rateMap: Record<string, number>;
 
   constructor(opts: RateLimiterOpts = {}) {
-    this.defaultRps = opts.defaultRps ?? 2;
-    this.burstSize = opts.burstSize ?? 5;
+    this.defaultRps = opts.defaultRps ?? (parseFloat(process.env.CRAWLER_RATE_LIMIT_DEFAULT_RPS ?? '') || 2);
+    this.burstSize = opts.burstSize ?? (parseInt(process.env.CRAWLER_RATE_LIMIT_BURST_SIZE ?? '') || 5);
     this.rateMap = opts.rateMap ?? {};
   }
 

package/src/sitemap.ts

@@ -13,7 +13,12 @@ import { logger } from './logger.js';
 const FETCH_TIMEOUT_MS = 10_000;
 const MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)
 const MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index
-const MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory
+
+async function getSitemapMaxUrls(): Promise<number> {
+  const { getSystemConfig } = await import('@detiq/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return parseInt(String(cfg?.sitemapMaxUrls ?? '5000'), 10);
+}
 
 function toArray<T>(x: T | T[] | undefined | null): T[] {
   if (x == null) return [];
@@ -44,7 +49,7 @@ export function sameSite(candidateUrl: string, siteOrigin: string): boolean {
  * and deduped. Pure function — no I/O — for direct unit testing without a
  * real HTTP fetch.
  */
-export function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {
+export function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls = 5000): { pageUrls: string[]; subSitemapUrls: string[] } {
   const rawPageUrls: string[] = [];
   let subSitemapUrls: string[] = [];
 
@@ -66,7 +71,7 @@ export function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pag
   // Same-site only (www/protocol-tolerant) — a sitemap listing external URLs shouldn't seed this project's queue.
   const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
 
-  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };
 }
 
 async function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {
@@ -103,22 +108,23 @@ async function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | nu
  * Fetches and parses a sitemap (or sitemap index, recursed up to a small
  * depth) into a flat, deduped list of same-origin URLs.
  */
-export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {
+export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any, maxUrls?: number): Promise<string[]> {
   if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  if (maxUrls === undefined) maxUrls = await getSitemapMaxUrls();
   const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
   if (!parsed) return [];
 
-  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);
   const collected = [...pageUrls];
 
-  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {
     const subResults = await Promise.all(
-      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))
     );
     for (const sub of subResults) collected.push(...sub);
   }
 
-  return [...new Set(collected)].slice(0, MAX_URLS);
+  return [...new Set(collected)].slice(0, maxUrls);
 }
 
 /**
@@ -192,6 +198,7 @@ export async function discoverProjectUrls(
   try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }
 
   const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+  const maxUrls = await getSitemapMaxUrls();
 
   // 1. robots.txt → declared sitemaps
   let robotsSitemaps: string[] = [];
@@ -216,11 +223,11 @@ export async function discoverProjectUrls(
   const seen = new Map<string, UrlCandidate>();
   let sitemapFound = false;
   for (const sm of sitemapTargets) {
-    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);
     if (urls.length) sitemapFound = true;
     const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';
     for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
   }
 
-  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
+  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };
 }

package/src/url-pattern-completer.ts

@@ -70,7 +70,7 @@ Respond with ONLY valid JSON, no explanation:
     const Anthropic = (await import('@anthropic-ai/sdk')).default;
     const client = new Anthropic({ apiKey: llmCfg.apiKey });
     const response = await client.messages.create({
-      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+      model: llmCfg.model ?? process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
       temperature: 0,
       // @ts-ignore: cache_control supported in beta

package/src/visual-diff-verdict.ts

@@ -23,9 +23,11 @@ export function pHashVerdict(
     : changeLevel === 'MINOR' ? 'MINOR'
       : changeLevel === 'MODERATE' ? 'SIGNIFICANT'
         : 'CRITICAL';
+  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? '') || 5;
+  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? '') || 20;
   const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'
-    : pHashDistance < 5 ? 'APPROVE'
-      : pHashDistance < 20 ? 'REVIEW'
+    : pHashDistance < approveThreshold ? 'APPROVE'
+      : pHashDistance < reviewThreshold ? 'REVIEW'
         : 'FLAG';
   return { changeType, recommendation, changes };
 }