@detiq/crawler 1.1.11 → 1.1.13

package/src/profiles/domain-profile.ts

@@ -114,7 +114,7 @@ const PROFILES: Record<Industry, Omit<DomainProfile, 'confidence' | 'evidence'>>
   },
 };
 
-const INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
+const BASE_INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
   { industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
   { industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
   { industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
@@ -125,6 +125,29 @@ const INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
   { industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },
 ];
 
+// Allow additional industry hint keywords via ENV without replacing the baseline.
+// Format: JSON array of { industry: string; terms: string[] } — terms are treated as case-insensitive regex.
+// Example: EXTRA_INDUSTRY_HINTS_JSON='[{"industry":"saas","terms":["platform","multitenant"]}]'
+const _extraHints: Array<{ industry: Industry; terms: RegExp[] }> = (() => {
+  const raw = process.env.EXTRA_INDUSTRY_HINTS_JSON;
+  if (!raw) return [];
+  try {
+    const parsed: Array<{ industry: string; terms: string[] }> = JSON.parse(raw);
+    return parsed.map(h => ({ industry: h.industry as Industry, terms: h.terms.map(t => new RegExp(t, 'i')) }));
+  } catch { return []; }
+})();
+
+const INDUSTRY_HINTS = [...BASE_INDUSTRY_HINTS, ..._extraHints];
+
+// Allow patching profile fields (riskyActions, piiRules, etc.) per industry via ENV.
+// Format: JSON object keyed by industry, value is partial DomainProfile fields to merge.
+// Example: DOMAIN_PROFILE_OVERRIDES_JSON='{"saas":{"riskyActions":["delete workspace","export all data"]}}'
+const _profileOverrides: Partial<Record<Industry, Partial<Omit<DomainProfile, 'confidence' | 'evidence'>>>> = (() => {
+  const raw = process.env.DOMAIN_PROFILE_OVERRIDES_JSON;
+  if (!raw) return {};
+  try { return JSON.parse(raw); } catch { return {}; }
+})();
+
 export function classifyDomainProfile(input: {
   appUrl: string;
   projectName?: string | null;

package/src/rate-limiter.ts

@@ -30,8 +30,8 @@ export class DomainRateLimiter {
   private readonly rateMap: Record<string, number>;
 
   constructor(opts: RateLimiterOpts = {}) {
-    this.defaultRps = opts.defaultRps ?? 2;
-    this.burstSize = opts.burstSize ?? 5;
+    this.defaultRps = opts.defaultRps ?? (parseFloat(process.env.CRAWLER_RATE_LIMIT_DEFAULT_RPS ?? '') || 2);
+    this.burstSize = opts.burstSize ?? (parseInt(process.env.CRAWLER_RATE_LIMIT_BURST_SIZE ?? '') || 5);
     this.rateMap = opts.rateMap ?? {};
   }
 

package/src/test-executor.ts

@@ -138,6 +138,116 @@ function applyHealInfo(result: StepResult, r: { healed: boolean; healedVia?: 'me
   result.healConfidence = r.healConfidence;
 }
 
+export async function executeExecutionStep(
+  page: Page,
+  step: ExecutionStep,
+  options: { healCtx?: { tenantId: string; projectId: string } } = {},
+): Promise<Pick<StepResult, 'healed' | 'healedVia' | 'healReasoning' | 'healConfidence' | 'elementBoundingRect' | 'elementSelector'>> {
+  const result: Pick<StepResult, 'healed' | 'healedVia' | 'healReasoning' | 'healConfidence' | 'elementBoundingRect' | 'elementSelector'> = {};
+  const timeout = step.timeout ?? parseInt(process.env.TEST_STEP_TIMEOUT_MS ?? '8000', 10);
+
+  switch (step.action) {
+    case 'navigate':
+      if (!step.url) throw new Error('Navigate step missing url');
+      assertCrawlUrlSafe(step.url);
+      await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout });
+      break;
+
+    case 'click': {
+      const r = await resolveLocator(page, step, options.healCtx);
+      if (!r) throw new Error('Could not resolve locator');
+      result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
+      result.elementSelector = step.selector;
+      await r.loc.first().click({ timeout });
+      applyHealInfo(result as StepResult, r);
+      break;
+    }
+
+    case 'fill': {
+      const r = await resolveLocator(page, step, options.healCtx);
+      if (!r) throw new Error('Could not resolve locator');
+      result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
+      result.elementSelector = step.selector;
+      await r.loc.first().fill(step.value ?? '', { timeout });
+      applyHealInfo(result as StepResult, r);
+      break;
+    }
+
+    case 'select': {
+      const r = await resolveLocator(page, step, options.healCtx);
+      if (!r) throw new Error('Could not resolve locator');
+      result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
+      result.elementSelector = step.selector;
+      await r.loc.first().selectOption(step.value ?? '', { timeout });
+      applyHealInfo(result as StepResult, r);
+      break;
+    }
+
+    case 'hover': {
+      const r = await resolveLocator(page, step, options.healCtx);
+      if (!r) throw new Error('Could not resolve locator');
+      result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
+      result.elementSelector = step.selector;
+      await r.loc.first().hover({ timeout });
+      applyHealInfo(result as StepResult, r);
+      break;
+    }
+
+    case 'press_key':
+      await page.keyboard.press(step.value ?? 'Enter');
+      break;
+
+    case 'assert_visible': {
+      const r = await resolveLocator(page, step, options.healCtx);
+      if (!r) throw new Error('Could not resolve locator');
+      await r.loc.first().waitFor({ state: 'visible', timeout });
+      result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
+      result.elementSelector = step.selector;
+      applyHealInfo(result as StepResult, r);
+      break;
+    }
+
+    case 'assert_text': {
+      const r = await resolveLocator(page, step, options.healCtx);
+      if (!r) throw new Error('Could not resolve locator');
+      const text = await r.loc.first().textContent({ timeout });
+      if (!text?.includes(step.value ?? '')) {
+        throw new Error(`Expected text "${step.value}" not found. Got: "${text?.slice(0, 100)}"`);
+      }
+      result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
+      result.elementSelector = step.selector;
+      applyHealInfo(result as StepResult, r);
+      break;
+    }
+
+    case 'assert_url': {
+      const url = page.url();
+      if (!url.includes(step.value ?? '')) {
+        throw new Error(`Expected URL to contain "${step.value}". Got: "${url}"`);
+      }
+      break;
+    }
+
+    case 'wait':
+      await page.waitForTimeout(step.timeout ?? parseInt(process.env.TEST_STEP_DELAY_MS ?? '2000', 10));
+      break;
+
+    case 'screenshot':
+      break;
+
+    case 'script':
+    case 'setVariable':
+      if (step.value) {
+        await page.evaluate(step.value).catch(err => {
+          throw new Error(`Script execution failed: ${err.message}`);
+        });
+      }
+      break;
+  }
+
+  return result;
+}
+
 async function runPlan(
   page: Page,
   plan: ExecutionStep[],
@@ -253,6 +363,18 @@ async function runPlan(
           result.screenshotPath = ssPath;
           break;
         }
+
+        case 'script':
+        case 'setVariable': {
+          if (step.value) {
+            // Execute arbitrary JS string. Often used by manual test builder extension for custom variables
+            // or assertions.
+            await page.evaluate(step.value).catch(err => {
+              throw new Error(`Script execution failed: ${err.message}`);
+            });
+          }
+          break;
+        }
       }
 
       result.status = 'PASSED';

package/src/url-pattern-completer.ts

@@ -70,7 +70,7 @@ Respond with ONLY valid JSON, no explanation:
     const Anthropic = (await import('@anthropic-ai/sdk')).default;
     const client = new Anthropic({ apiKey: llmCfg.apiKey });
     const response = await client.messages.create({
-      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+      model: llmCfg.model ?? process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
       temperature: 0,
       // @ts-ignore: cache_control supported in beta

package/src/visual-diff-verdict.ts

@@ -23,9 +23,11 @@ export function pHashVerdict(
     : changeLevel === 'MINOR' ? 'MINOR'
       : changeLevel === 'MODERATE' ? 'SIGNIFICANT'
         : 'CRITICAL';
+  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? '') || 5;
+  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? '') || 20;
   const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'
-    : pHashDistance < 5 ? 'APPROVE'
-      : pHashDistance < 20 ? 'REVIEW'
+    : pHashDistance < approveThreshold ? 'APPROVE'
+      : pHashDistance < reviewThreshold ? 'REVIEW'
         : 'FLAG';
   return { changeType, recommendation, changes };
 }

package/types/crawler.d.ts

@@ -190,4 +190,13 @@ export declare function crawlProject(opts: {
     domHashes: string[];
     discoveredUrls?: undefined;
 }>;
+export declare function domFallbackElements(page: any): Promise<Array<{
+    meaning: string;
+    role: string;
+    expectedData?: string;
+    notes?: string;
+    boundingRect?: Record<string, number>;
+    ariaState?: Record<string, string>;
+    parentLandmark?: string;
+}>>;
 export {};

package/types/locator-extractor.d.ts

@@ -17,6 +17,7 @@ export declare function elementToPlaywrightLocator(element: {
     text?: string | null;
     notes?: unknown;
     selector?: string | null;
+    ariaState?: Record<string, unknown> | null;
 }): PlaywrightLocator | null;
 /**
  * Convert an array of DB Element records to deduplicated Playwright locators.
@@ -29,6 +30,7 @@ export declare function elementsToPlaywrightLocators(elements: Array<{
     notes?: unknown;
     selector?: string | null;
     isInteractive?: boolean | null;
+    ariaState?: Record<string, unknown> | null;
 }>): PlaywrightLocator[];
 /**
  * Format locators as a block for injection into LLM script generation prompts.

package/types/test-executor.d.ts

@@ -1,3 +1,4 @@
+import { type Page } from 'playwright';
 import type { ExecutionStep } from '@detiq/agents';
 export interface StepResult {
     stepNumber: number;
@@ -47,6 +48,12 @@ export interface ExecutionResult {
         duration: number;
     }>;
 }
+export declare function executeExecutionStep(page: Page, step: ExecutionStep, options?: {
+    healCtx?: {
+        tenantId: string;
+        projectId: string;
+    };
+}): Promise<Pick<StepResult, 'healed' | 'healedVia' | 'healReasoning' | 'healConfidence' | 'elementBoundingRect' | 'elementSelector'>>;
 export declare function executeTestPlan(plan: ExecutionStep[], options: {
     screenshotDir: string;
     testRunId: string;