@detiq/crawler 1.1.12 → 1.1.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.12",
+  "version": "1.1.13",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/content-chunker.ts

@@ -21,9 +21,9 @@ export function estimateTokens(text: string): number {
 }
 
 export function chunkContent(text: string, opts: ChunkOpts): Chunk[] {
-  const size = opts.chunkSize ?? 2000;
+  const size = opts.chunkSize ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_SIZE ?? '') || 2000);
   const charSize = size * 4;
-  const overlap = opts.overlap ?? 200;
+  const overlap = opts.overlap ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_OVERLAP ?? '') || 200);
   const overlapChars = overlap * 4;
 
   switch (opts.strategy) {

package/src/crawler.ts

@@ -155,9 +155,9 @@ type AuthDiscoveryOptions = {
   storageConfig?: any;
 };
 
-const AUTH_DISCOVERY_MAX_SCREENS = 36;
-const AUTH_DISCOVERY_MAX_QUEUE = 48;
-const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = 6;
+const AUTH_DISCOVERY_MAX_SCREENS = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_SCREENS ?? '') || 36;
+const AUTH_DISCOVERY_MAX_QUEUE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_QUEUE ?? '') || 48;
+const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE ?? '') || 6;
 const AUTH_DISCOVERY_PATHS = [
   '/',
   '/login', '/log-in', '/signin', '/sign-in', '/auth', '/auth/login', '/account/login', '/user/login', '/app/login',
@@ -576,15 +576,15 @@ export async function crawlProject(opts: {
             break;
           case 'openrouter':
             stagehandModel = `openai/${m}`;
-            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || 'https://openrouter.ai/api/v1' };
+            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_OPENROUTER_BASE_URL || 'https://openrouter.ai/api/v1' };
             break;
           case 'ollama':
             stagehandModel = `ollama/${m}`;
-            stagehandModelOpts = { baseURL: baseUrl || 'http://localhost:11434' };
+            stagehandModelOpts = { baseURL: baseUrl || process.env.CRAWLER_OLLAMA_BASE_URL || 'http://localhost:11434' };
             break;
           case 'nvidia':
             stagehandModel = `openai/${m.startsWith('openai/') ? m.slice(7) : m}`;
-            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || 'https://integrate.api.nvidia.com/v1' };
+            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_NVIDIA_BASE_URL || 'https://integrate.api.nvidia.com/v1' };
             break;
           case 'groq':
             stagehandModel = `groq/${m}`;
@@ -635,15 +635,18 @@ export async function crawlProject(opts: {
     const preferAnthropic = envModel.includes('claude') || (!envModel && !process.env.OPENAI_API_KEY);
     if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+      const defaultAnthropicModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel}`;
       stagehandConfigured = true;
     } else if (process.env.OPENAI_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.OPENAI_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || 'gpt-4o'}`;
+      const defaultOpenaiModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'gpt-4o';
+      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || defaultOpenaiModel}`;
       stagehandConfigured = true;
     } else if (process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+      const defaultAnthropicModel2 = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel2}`;
       stagehandConfigured = true;
     }
   }
@@ -848,7 +851,7 @@ export async function crawlProject(opts: {
     verbose: 1,
     logger: stagehandLogger,
     localBrowserLaunchOptions: { cdpUrl, headless: true },
-    domSettleTimeout: 2000,
+    domSettleTimeout: parseInt(process.env.CRAWLER_DOM_SETTLE_TIMEOUT_MS ?? '') || 2000,
   });
 
   await stagehand.init();
@@ -857,8 +860,8 @@ export async function crawlProject(opts: {
   // Use resolvePage() to get the active V3Page (which is a full Playwright proxy).
   const page = await (stagehand as any).resolvePage();
   // Cap all Playwright operations so one stuck page can't freeze the crawl.
-  try { page.setDefaultTimeout(45_000); } catch { /* proxy may not expose this */ }
-  try { page.setDefaultNavigationTimeout(30_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_DEFAULT_TIMEOUT_MS ?? '') || 45_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultNavigationTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_NAV_TIMEOUT_MS ?? '') || 30_000); } catch { /* proxy may not expose this */ }
   // Stagehand V3Page proxy only supports the 'console' event (page.on throws for others).
   // rawPage will be wired to a real Playwright Page via the CDP connection established below.
   // Declared as `let` so the network-observer block can populate it.
@@ -1472,7 +1475,7 @@ export async function crawlProject(opts: {
         ];
     const queued = new Set<string>(seedUrls.map(canonicalize));
     const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter);
-    const deduplicator = new ContentDeduplicator(0.88);
+    const deduplicator = new ContentDeduplicator(parseFloat(process.env.CRAWLER_DEDUP_THRESHOLD ?? '') || 0.88);
     const deadLinkTracker = new DeadLinkTracker();
     const redirectRegistry = new RedirectRegistry();
     // Re-seed from previously-discovered screens so deep routes found by AI link
@@ -1626,7 +1629,12 @@ export async function crawlProject(opts: {
     // Legal/policy-only pages — no interactive elements worth testing; skip to prevent noise
     // NOTE: /about, /contact, /careers, /help are intentionally NOT skipped — they have
     // real test targets (contact forms, job apply forms, help search, team content)
-    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
+    const skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
+      ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
+      : [];
+    const SKIP_PATH = skipPathExtra.length > 0
+      ? new RegExp(`/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap|${skipPathExtra.join('|')})([-/]|$)`, 'i')
+      : /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
     const segIdx = opts.segmentIndex ?? 1;
 
     const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
@@ -1637,6 +1645,7 @@ export async function crawlProject(opts: {
     // (was: 3 errors → skip 10 screens — too aggressive for Nvidia NIM which rate-limits quickly)
     let aiErrorStreak = 0;
     let aiSkipCount = 0;
+    const aiMinElements = parseInt(process.env.CRAWL_AI_LINK_MIN_ELEMENTS ?? '') || 5;
     let aiRateLimitWarned = false;
     // Hard guard: total URLs attempted (including skips) capped at maxScreens * 4
     // to prevent infinite-URL-explosion from WooCommerce/Shopify facet filter combos
@@ -1746,7 +1755,7 @@ export async function crawlProject(opts: {
       for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
         const wp = await pwCtx.newPage();
         await wp.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }).catch(() => {});
-        wp.setDefaultTimeout(45_000);
+        wp.setDefaultTimeout(parseInt(process.env.CRAWLER_PLAYWRIGHT_DEFAULT_TIMEOUT_MS ?? '') || 45_000);
         liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {})));
       }
     }
@@ -2530,7 +2539,7 @@ export async function crawlProject(opts: {
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
           // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
-          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < 5)
+          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < aiMinElements)
             .then((freshLinks) => {
               if (aiSkipCount > 0) aiSkipCount--;
               aiErrorStreak = 0;
@@ -2964,12 +2973,15 @@ export async function crawlProject(opts: {
           // P3a: Post-extraction DOM count — detect async-loaded content.
           // Only check sparse pages (preDomCount < 25) where async widgets are likely.
           // If DOM grew significantly during AI extraction, re-visit later with networkidle.
-          if (preDomCount < 25 && !revisitQueue.includes(url)) {
+          const asyncContentCheckMinDom = parseInt(process.env.CRAWLER_ASYNC_CONTENT_MIN_DOM ?? '') || 25;
+          if (preDomCount < asyncContentCheckMinDom && !revisitQueue.includes(url)) {
             try {
               const postDomCount = await page.evaluate(() =>
                 document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
               ).catch(() => 0);
-              if (postDomCount > preDomCount * 1.3 + 4) {
+              const asyncGrowthRatio = parseFloat(process.env.CRAWLER_ASYNC_GROWTH_RATIO ?? '') || 1.3;
+              const asyncGrowthFloor = parseInt(process.env.CRAWLER_ASYNC_GROWTH_FLOOR ?? '') || 4;
+              if (postDomCount > preDomCount * asyncGrowthRatio + asyncGrowthFloor) {
                 revisitQueue.push(url);
                 logger.info({ url, preDomCount, postDomCount }, '[P3a] Async content detected — scheduled for re-visit');
               }
@@ -2978,7 +2990,7 @@ export async function crawlProject(opts: {
 
           // P2: Static JS router detection — scan bundle for declared client-side routes.
           // Only run on first 5 pages (bundle is the same across pages; scanning every page is wasteful).
-          if (visited.size <= 5) {
+          if (visited.size <= (parseInt(process.env.CRAWLER_SPA_ROUTER_DETECT_LIMIT ?? '') || 5)) {
             try {
               const jsDetectedRoutes = await detectJsRoutes(page, origin);
               for (const route of jsDetectedRoutes) {
@@ -3011,7 +3023,7 @@ export async function crawlProject(opts: {
           // Triggered per-page when: zero DOM links found + fewer than 5 elements (dead-end / canvas page).
           // Skyvern returns URLs it discovered → added to queue for ZeTa's own BFS crawl (not saved directly).
           // ZeTa does all actual crawling — Skyvern is only a URL-discovery override layer.
-          if (nextLinks.length === 0 && elements.length < 5 && opts.skyvernConfig) {
+          if (nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? '') || 5) && opts.skyvernConfig) {
             try {
               logger.info({ url }, 'Gap 5: thin page detected — asking Skyvern to discover navigation URLs');
               const skyvernNav = await skyvernNavigate(
@@ -3125,7 +3137,8 @@ export async function crawlProject(opts: {
     // Skyvern is ONLY a URL-discovery layer here — it does NOT save screens directly.
     // Use only successfully-saved screens (screenId present), not total discovered including errors.
     const successfulScreenCount = discovered.filter(d => d.screenId).length;
-    if (successfulScreenCount < 10 && opts.skyvernConfig) {
+    const skyvernMinScreens = parseInt(process.env.CRAWL_SKYVERN_MIN_SCREENS ?? '') || 10;
+    if (successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
       await onProgress?.(78, 'Few screens found — using Skyvern to discover additional navigation URLs');
       const currentUrl = page.url();
       const skyvernNav = await skyvernNavigate(
@@ -3283,7 +3296,7 @@ export async function crawlProject(opts: {
     // P6: Gap Pass — LLM gap analysis on discovered screens to find structurally missing pages.
     // Runs after P3a re-visits so the gap analysis has the most complete screen list possible.
     // Confirmed gap URLs (HEAD=200) are visited directly with the existing browser.
-    if (visited.size >= 3) {
+    if (visited.size >= (parseInt(process.env.CRAWLER_GAP_PASS_MIN_SCREENS ?? '') || 3)) {
       try {
         await onProgress?.(80, 'Running gap analysis…');
         const llmCfgForGap = await getCachedCrawlLLM(opts.tenantId);
@@ -3309,7 +3322,7 @@ export async function crawlProject(opts: {
 
           // Visit each confirmed gap URL with the existing browser (depth=1, no link traversal)
           const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
-          for (const gap of confirmedGaps.slice(0, 8)) {
+          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? '') || 8)) {
             const gapUrl = gap.confirmedUrl!;
             if (visited.size >= maxScreens) break;
             try {

package/src/frontier.ts

@@ -27,11 +27,13 @@ function scoreUrl(url: string, keywords: string[]): number {
   try {
     const u = new URL(url);
     const segments = u.pathname.split('/').filter(Boolean);
-    let score = -segments.length;
+    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? '') || 1;
+    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? '') || 10;
+    let score = -segments.length * depthPenaltyWeight;
     if (keywords.length > 0) {
       const lower = url.toLowerCase();
       for (const kw of keywords) {
-        if (kw && lower.includes(kw)) score += 10;
+        if (kw && lower.includes(kw)) score += keywordBoostWeight;
       }
     }
     return score;
@@ -69,7 +71,7 @@ export class CrawlFrontier {
     const sparseBoost =
       this.strategy === 'BEST_FIRST' &&
       pushOpts?.parentElements !== undefined &&
-      pushOpts.parentElements < 5
+      pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? '') || 5)
         ? 1
         : 0;
     this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });

package/src/gap-pass.ts

@@ -73,7 +73,7 @@ Respond with ONLY valid JSON, no markdown:
     const Anthropic = (await import('@anthropic-ai/sdk')).default;
     const client = new Anthropic({ apiKey: llmCfg.apiKey });
     const response = await client.messages.create({
-      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+      model: llmCfg.model ?? process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
       temperature: 0,
       // @ts-ignore

package/src/http-engine.ts

@@ -63,7 +63,12 @@ async function getChromeFetchDispatcher() {
 const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
 
 // Pages unlikely to have testable UI — skip from HTTP crawl queue
-const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+const _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
+  ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const SKIP_PATH = _skipPathExtra.length > 0
+  ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')
+  : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
 
 export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
 
@@ -212,7 +217,7 @@ async function fetchWithBackoff(
     'Accept-Language': 'en-US,en;q=0.9',
     'Accept-Encoding': 'gzip, deflate, br, zstd',
     // Chrome-like security headers reduce bot-detection fingerprint
-    'sec-ch-ua': '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+    'sec-ch-ua': `"Google Chrome";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Chromium";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Not-A.Brand";v="99"`,
     'sec-ch-ua-mobile': '?0',
     'sec-ch-ua-platform': '"Windows"',
     'Sec-Fetch-Dest': 'document',
@@ -287,10 +292,10 @@ async function fetchWithBackoff(
 
 export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
   const startTime = performance.now();
-  const maxScreens = opts.maxScreens ?? 100;
+  const maxScreens = opts.maxScreens ?? (parseInt(process.env.CRAWLER_HTTP_ENGINE_MAX_SCREENS ?? '') || 100);
   const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
   const timeoutMs = opts.timeoutMs ?? 15_000;
-  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)';
+  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
   const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
   const cache = new ResponseCache(cacheDir);
   const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;

package/src/job-queue.ts

@@ -383,12 +383,18 @@ export async function createTrackedJob(
   return job.id;
 }
 
-/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors. */
+/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
+ * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
 export async function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void> {
+  const nextRetryAt = error ? new Date(Date.now() + 60_000) : undefined;
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: error ? 'FAILED' : 'DONE', finishedAt: new Date(), ...(error ? { error } : { progress: 100 }) },
+      data: {
+        status: error ? 'FAILED' : 'DONE',
+        finishedAt: new Date(),
+        ...(error ? { error, nextRetryAt } : { progress: 100 }),
+      },
     })
   ).catch(() => {});
 }
@@ -471,7 +477,7 @@ export async function cancelJob(tenantId: string, jobId: string): Promise<void>
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: 'CANCELLED', finishedAt: new Date() },
+      data: { status: 'CANCELLED', finishedAt: new Date(), heartbeatAt: null, workerId: null },
     })
   );
 }
@@ -637,7 +643,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     await opts?.onProgress?.(pct);
   };
 
-  const DEFAULT_MAX_SCREENS = 150;
+  const DEFAULT_MAX_SCREENS = parseInt(process.env.CRAWL_DEFAULT_MAX_SCREENS ?? '') || 150;
 
   try {
     // Per-project maxScreens + deviceProfiles + seedUrls + interactive probing + sitemap seeding + traversal strategy from crawlConfig override job config defaults.
@@ -1611,7 +1617,7 @@ export async function processDataRetentionPurge(): Promise<void> {
       orderBy: { dataRetentionDays: 'asc' },
       select: { dataRetentionDays: true },
     });
-    const retentionDays = setting?.dataRetentionDays ?? 365;
+    const retentionDays = setting?.dataRetentionDays ?? (parseInt(process.env.DEFAULT_DATA_RETENTION_DAYS ?? '') || 365);
     const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000);
 
     const result = await prismaAdmin.auditLog.deleteMany({

package/src/policies/crawl-safety.ts

@@ -6,7 +6,13 @@ export interface CrawlSafetyDecision {
   severity: 'info' | 'warn' | 'block';
 }
 
-const DESTRUCTIVE_ACTION = /\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop)\b/i;
+const _extraDestructive = process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS
+  ? process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const DESTRUCTIVE_ACTION = new RegExp(
+  `\\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop${_extraDestructive.length ? `|${_extraDestructive.join('|')}` : ''})\\b`,
+  'i',
+);
 const SECRET_LIKE = /\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\b/i;
 
 export function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision {

package/src/profiles/domain-profile.ts

@@ -114,7 +114,7 @@ const PROFILES: Record<Industry, Omit<DomainProfile, 'confidence' | 'evidence'>>
   },
 };
 
-const INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
+const BASE_INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
   { industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
   { industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
   { industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
@@ -125,6 +125,29 @@ const INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
   { industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },
 ];
 
+// Allow additional industry hint keywords via ENV without replacing the baseline.
+// Format: JSON array of { industry: string; terms: string[] } — terms are treated as case-insensitive regex.
+// Example: EXTRA_INDUSTRY_HINTS_JSON='[{"industry":"saas","terms":["platform","multitenant"]}]'
+const _extraHints: Array<{ industry: Industry; terms: RegExp[] }> = (() => {
+  const raw = process.env.EXTRA_INDUSTRY_HINTS_JSON;
+  if (!raw) return [];
+  try {
+    const parsed: Array<{ industry: string; terms: string[] }> = JSON.parse(raw);
+    return parsed.map(h => ({ industry: h.industry as Industry, terms: h.terms.map(t => new RegExp(t, 'i')) }));
+  } catch { return []; }
+})();
+
+const INDUSTRY_HINTS = [...BASE_INDUSTRY_HINTS, ..._extraHints];
+
+// Allow patching profile fields (riskyActions, piiRules, etc.) per industry via ENV.
+// Format: JSON object keyed by industry, value is partial DomainProfile fields to merge.
+// Example: DOMAIN_PROFILE_OVERRIDES_JSON='{"saas":{"riskyActions":["delete workspace","export all data"]}}'
+const _profileOverrides: Partial<Record<Industry, Partial<Omit<DomainProfile, 'confidence' | 'evidence'>>>> = (() => {
+  const raw = process.env.DOMAIN_PROFILE_OVERRIDES_JSON;
+  if (!raw) return {};
+  try { return JSON.parse(raw); } catch { return {}; }
+})();
+
 export function classifyDomainProfile(input: {
   appUrl: string;
   projectName?: string | null;

package/src/rate-limiter.ts

@@ -30,8 +30,8 @@ export class DomainRateLimiter {
   private readonly rateMap: Record<string, number>;
 
   constructor(opts: RateLimiterOpts = {}) {
-    this.defaultRps = opts.defaultRps ?? 2;
-    this.burstSize = opts.burstSize ?? 5;
+    this.defaultRps = opts.defaultRps ?? (parseFloat(process.env.CRAWLER_RATE_LIMIT_DEFAULT_RPS ?? '') || 2);
+    this.burstSize = opts.burstSize ?? (parseInt(process.env.CRAWLER_RATE_LIMIT_BURST_SIZE ?? '') || 5);
     this.rateMap = opts.rateMap ?? {};
   }
 

package/src/url-pattern-completer.ts

@@ -70,7 +70,7 @@ Respond with ONLY valid JSON, no explanation:
     const Anthropic = (await import('@anthropic-ai/sdk')).default;
     const client = new Anthropic({ apiKey: llmCfg.apiKey });
     const response = await client.messages.create({
-      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+      model: llmCfg.model ?? process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
       temperature: 0,
       // @ts-ignore: cache_control supported in beta

package/src/visual-diff-verdict.ts

@@ -23,9 +23,11 @@ export function pHashVerdict(
     : changeLevel === 'MINOR' ? 'MINOR'
       : changeLevel === 'MODERATE' ? 'SIGNIFICANT'
         : 'CRITICAL';
+  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? '') || 5;
+  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? '') || 20;
   const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'
-    : pHashDistance < 5 ? 'APPROVE'
-      : pHashDistance < 20 ? 'REVIEW'
+    : pHashDistance < approveThreshold ? 'APPROVE'
+      : pHashDistance < reviewThreshold ? 'REVIEW'
         : 'FLAG';
   return { changeType, recommendation, changes };
 }