@detiq/crawler 1.0.4 → 1.0.5

package/dist/crawler.d.ts

@@ -35,10 +35,24 @@ export declare function crawlProject(opts: {
     tenantId: string;
     projectId: string;
     appUrl: string;
+    /** The viewport used for this crawl pass */
+    viewport?: 'DESKTOP' | 'MOBILE_PORTRAIT' | 'TABLET';
     credentials?: Credentials;
     /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
     storageState?: string;
     screenshotDir?: string;
+    /** Cloud storage config for immediate screenshot upload. If set, screenshots are uploaded to cloud immediately after capture and cloudUrl is persisted. */
+    storageConfig?: {
+        provider: string;
+        bucket: string;
+        region?: string;
+        accountId?: string;
+        endpointUrl?: string;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+        prefix?: string;
+        publicUrl?: string;
+    } | null;
     /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
     discoveryMode?: boolean;
     maxScreens?: number;

package/dist/crawler.js

@@ -49,6 +49,13 @@ import { screensToSitemapUrls, writeSitemap } from "./sitemap-writer.js";
 import { generateOpenApiSpec, specToYaml } from "./openapi-generator.js";
 import { WebhookNotifier } from "./webhook-notifier.js";
 import { generatePlaywrightTest } from "./playwright-recorder.js";
+const _crawlLLMCache = /* @__PURE__ */ new Map();
+async function getCachedCrawlLLM(tenantId) {
+  if (_crawlLLMCache.has(tenantId)) return _crawlLLMCache.get(tenantId);
+  const cfg = await getTaskAgentLLMConfig(tenantId, "crawl").catch(() => null);
+  _crawlLLMCache.set(tenantId, cfg);
+  return cfg;
+}
 function getGitInfo() {
   try {
     const branch = execSync("git rev-parse --abbrev-ref HEAD", { encoding: "utf8", timeout: 3e3 }).trim();
@@ -325,39 +332,104 @@ async function crawlProject(opts, onProgress) {
     }
   };
   const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.env.NODE_ENV === "production";
-  const toStagehandModel = (model, provider) => model.includes("/") ? model : `${provider}/${model}`;
-  let stagehandModel = toStagehandModel(config.llmModel, "openai");
-  let stagehandApiKey = "";
+  let stagehandModel = config.llmModel.includes("/") ? config.llmModel : `openai/${config.llmModel}`;
+  let stagehandModelOpts = {};
+  let stagehandConfigured = false;
   let configuredProvider = "";
   try {
     const crawlLLM = await getTaskAgentLLMConfig(opts.tenantId, "crawl");
     if (crawlLLM) {
       configuredProvider = crawlLLM.provider;
-      if (crawlLLM.apiKey && crawlLLM.model) {
-        if (crawlLLM.provider === "anthropic" || crawlLLM.provider === "openai") {
-          stagehandModel = toStagehandModel(crawlLLM.model, crawlLLM.provider);
-          stagehandApiKey = crawlLLM.apiKey;
-        } else if (crawlLLM.provider === "openrouter") {
-          const model = crawlLLM.model.includes("/") ? crawlLLM.model : `openai/${crawlLLM.model}`;
-          stagehandModel = model;
-          stagehandApiKey = crawlLLM.apiKey;
+      const hasKey = crawlLLM.apiKey !== void 0 && crawlLLM.apiKey !== null;
+      const apiKey = hasKey ? crawlLLM.apiKey : void 0;
+      const baseUrl = crawlLLM.baseUrl || void 0;
+      const m = crawlLLM.model;
+      if (m && (hasKey || crawlLLM.provider === "ollama" || crawlLLM.provider === "bedrock" || crawlLLM.provider === "vertex")) {
+        stagehandConfigured = true;
+        switch (crawlLLM.provider) {
+          case "anthropic":
+            stagehandModel = m.includes("/") ? m : `anthropic/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "openai":
+            stagehandModel = m.includes("/") ? m : `openai/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "gemini":
+            stagehandModel = `google/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "grok":
+            stagehandModel = `xai/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "openrouter":
+            stagehandModel = `openai/${m}`;
+            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, baseURL: baseUrl || "https://openrouter.ai/api/v1" };
+            break;
+          case "ollama":
+            stagehandModel = `ollama/${m}`;
+            stagehandModelOpts = { baseURL: baseUrl || "http://localhost:11434" };
+            break;
+          case "nvidia":
+            stagehandModel = `openai/${m.startsWith("openai/") ? m.slice(7) : m}`;
+            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, baseURL: baseUrl || "https://integrate.api.nvidia.com/v1" };
+            break;
+          case "groq":
+            stagehandModel = `groq/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "mistral":
+            stagehandModel = `mistral/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "deepseek":
+            stagehandModel = `deepseek/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "perplexity":
+            stagehandModel = `perplexity/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "togetherai":
+            stagehandModel = `togetherai/${m}`;
+            stagehandModelOpts = apiKey ? { apiKey } : {};
+            break;
+          case "bedrock":
+            stagehandModel = `bedrock/${m}`;
+            stagehandModelOpts = {};
+            break;
+          case "vertex":
+            stagehandModel = `vertex/${m}`;
+            stagehandModelOpts = {};
+            break;
+          case "azure":
+            stagehandModel = `azure/${m}`;
+            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, ...baseUrl ? { baseURL: baseUrl } : {} };
+            break;
+          default:
+            stagehandModel = `openai/${m}`;
+            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, ...baseUrl ? { baseURL: baseUrl } : {} };
         }
       }
     }
   } catch {
   }
-  if (!stagehandApiKey) {
+  if (!stagehandConfigured) {
     const envModel = process.env.STAGEHAND_MODEL ?? "";
     const preferAnthropic = envModel.includes("claude") || !envModel && !process.env.OPENAI_API_KEY;
     if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
-      stagehandApiKey = process.env.ANTHROPIC_API_KEY;
+      stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
       stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || "claude-sonnet-4-6"}`;
+      stagehandConfigured = true;
     } else if (process.env.OPENAI_API_KEY) {
-      stagehandApiKey = process.env.OPENAI_API_KEY;
+      stagehandModelOpts = { apiKey: process.env.OPENAI_API_KEY };
       stagehandModel = envModel.includes("/") ? envModel : `openai/${envModel || "gpt-4o"}`;
+      stagehandConfigured = true;
     } else if (process.env.ANTHROPIC_API_KEY) {
-      stagehandApiKey = process.env.ANTHROPIC_API_KEY;
+      stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
       stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || "claude-sonnet-4-6"}`;
+      stagehandConfigured = true;
     }
   }
   const elementLLM = await AppBrain.getTaskLLMConfig(opts.tenantId, "elementExtraction").catch(() => null);
@@ -390,10 +462,9 @@ async function crawlProject(opts, onProgress) {
     type: "CRAWL_PROFILE",
     payload: { profile: domainProfile, routeDecision }
   }).catch((err) => console.warn("[crawler] profile evidence save failed:", err?.message ?? err));
-  if (!stagehandApiKey) {
-    const unsupported = configuredProvider && !["anthropic", "openai", "openrouter"].includes(configuredProvider);
+  if (!stagehandConfigured) {
     throw new Error(
-      unsupported ? `Browser crawling requires Anthropic or OpenAI. Your primary provider "${configuredProvider}" is not supported for crawling. Add a separate Crawl API Key in Workspace \u2192 Settings \u2192 AI Provider.` : "No AI provider configured. Go to Workspace \u2192 Settings \u2192 AI Provider to set up Anthropic or OpenAI for browser crawling."
+      configuredProvider ? `No API key configured for provider "${configuredProvider}". Go to Workspace \u2192 Settings \u2192 AI Provider.` : "No AI provider configured. Go to Workspace \u2192 Settings \u2192 AI Provider to set up an AI provider for browser crawling."
     );
   }
   const stealthArgs = [
@@ -529,7 +600,7 @@ async function crawlProject(opts, onProgress) {
   };
   const stagehand = new Stagehand({
     env: "LOCAL",
-    model: { modelName: stagehandModel, apiKey: stagehandApiKey },
+    model: { modelName: stagehandModel, ...stagehandModelOpts },
     serverCache: true,
     verbose: 1,
     logger: stagehandLogger,
@@ -1339,7 +1410,7 @@ async function crawlProject(opts, onProgress) {
           console.warn(`[crawler] connection failed for ${url} \u2014 skipping (chrome-error page)`);
           continue;
         }
-        await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+        await page.waitForLoadState("load", { timeout: 8e3 }).catch(() => {
         });
         await waitForStableDOM(page);
         if (!opts.discoveryMode) {
@@ -1798,10 +1869,11 @@ async function crawlProject(opts, onProgress) {
             return "/";
           }
         })()}`);
+        const pageDpr = await page.evaluate(() => window.devicePixelRatio ?? 1).catch(() => 1);
         const [, elements, nextLinks, atElements] = await Promise.all([
           page.screenshot({ path: shotPath, fullPage: true }).then(async () => {
             const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
-            return AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl);
+            return AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl, pageDpr);
           }).catch((err) => console.error("[crawler] page.screenshot failed:", err)),
           extractElements(stagehand, page, elementLLM ?? void 0).catch((err) => {
             console.error("[crawler] extractElements failed:", err);
@@ -1883,7 +1955,7 @@ async function crawlProject(opts, onProgress) {
               await fs.writeFile(mdPath, scrubbed, "utf8");
               const summaryPromise = (async () => {
                 try {
-                  const llmCfg = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => void 0);
+                  const llmCfg = await getCachedCrawlLLM(opts.tenantId);
                   const snippet = extracted.markdown.slice(0, 1200);
                   const { text, usage } = await askTracked(
                     "You are a QA analyst. Given page content, write ONE crisp sentence describing what this screen does or shows. Be specific. No preamble.",
@@ -2020,15 +2092,38 @@ ${snippet}`,
                     logger.warn({ screenId: screen.id, err: String(aiErr) }, "[crawler] AI visual compare failed, keeping pHash-only verdict");
                   }
                 }
-                const { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
+                let { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
+                if (changeType !== "NONE") {
+                  try {
+                    const project = await AppBrain.getProject(opts.tenantId, opts.projectId);
+                    const threshold = project?.visualDiffThreshold ?? null;
+                    if (threshold !== null && pHashDistance < threshold) {
+                      changeType = "NONE";
+                    }
+                  } catch {
+                  }
+                }
                 if (changeType !== "NONE") {
                   let newScreenshotCloudUrl;
+                  let diffImageCloudUrl;
                   try {
                     const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import("@detiq/app-brain");
                     const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
                     if (storageCfg) {
                       const key = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
                       newScreenshotCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, shotPath, key);
+                      try {
+                        const { generatePixelmatchDiff } = await import("./phash.js");
+                        const diffBuffer = await generatePixelmatchDiff(baseline.storagePath, shotPath, baseline.ignoredRegions ?? []);
+                        const diffTmpPath = `${shotPath}.diff.png`;
+                        await fs.writeFile(diffTmpPath, diffBuffer);
+                        const diffKey = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}-diff.png`, storageCfg.prefix);
+                        diffImageCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, diffTmpPath, diffKey);
+                        fs.unlink(diffTmpPath).catch(() => {
+                        });
+                      } catch (diffErr) {
+                        logger.warn({ screenId: screen.id, err: String(diffErr) }, "[crawler] pixelmatch diff generation failed, skipping");
+                      }
                     }
                   } catch {
                   }
@@ -2036,6 +2131,7 @@ ${snippet}`,
                     baselineId: baseline.id,
                     newScreenshotPath: shotPath,
                     newScreenshotCloudUrl,
+                    diffImageCloudUrl,
                     diffScore: pHashDistance,
                     changeType,
                     changes: allChanges,
@@ -2056,17 +2152,20 @@ ${snippet}`,
           })();
         }
         const linkMappings = await extractLinkMappings(page, origin);
-        for (const el of elements) {
-          const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark);
-          const label = el.meaning;
-          const role = el.role ?? "unknown";
-          const selectorHint = atByRole.get(role) ?? role;
-          const selectors = [
-            { type: "role-label", value: `${role}:${label}`, confidence: 0.9 },
-            { type: "aria-label", value: label, confidence: 0.85 },
-            { type: "tag-hint", value: selectorHint, confidence: 0.6 }
-          ];
-          AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
+        for (let _ei = 0; _ei < elements.length; _ei += 4) {
+          const _batch = elements.slice(_ei, _ei + 4);
+          await Promise.all(_batch.map(async (el) => {
+            const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark);
+            const label = el.meaning;
+            const role = el.role ?? "unknown";
+            const selectorHint = atByRole.get(role) ?? role;
+            const selectors = [
+              { type: "role-label", value: `${role}:${label}`, confidence: 0.9 },
+              { type: "aria-label", value: label, confidence: 0.85 },
+              { type: "tag-hint", value: selectorHint, confidence: 0.6 }
+            ];
+            AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
+          }));
         }
         for (const lm of linkMappings) {
           const safety = assessActionSafety(lm.label, domainProfile);
@@ -2366,7 +2465,7 @@ ${snippet}`,
     }
     try {
       const { runUrlPatternCompletion } = await import("./url-pattern-completer.js");
-      const llmCfgForPatterns = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => null);
+      const llmCfgForPatterns = await getCachedCrawlLLM(opts.tenantId);
       const urlPatternCandidates = await runUrlPatternCompletion(
         Array.from(visited.keys()),
         opts.appUrl,
@@ -2427,7 +2526,7 @@ ${snippet}`,
     if (visited.size >= 3) {
       try {
         await onProgress?.(80, "Running gap analysis\u2026");
-        const llmCfgForGap = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => null);
+        const llmCfgForGap = await getCachedCrawlLLM(opts.tenantId);
         const discoveredForGap = Array.from(visited.entries()).map(([url, screenId]) => {
           const d = discovered.find((dd) => dd.screenId === screenId);
           return { url, name: d ? new URL(url).pathname.split("/").filter(Boolean).pop() ?? "screen" : void 0, elements: d?.elements };
@@ -2876,7 +2975,41 @@ ${snippet}`,
 const ELEMENT_NOISE = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
 async function extractElements(stagehand, page, llmCfg) {
   const domElements = await domFallbackElements(page);
-  if (domElements.length >= 2) return domElements;
+  const iframeElements = [];
+  try {
+    const iframes = page.frames ? page.frames() : [];
+    for (const frame of iframes) {
+      if (frame === page.mainFrame?.()) continue;
+      const iframeUrl = await frame.url().catch(() => "");
+      if (!iframeUrl || iframeUrl === "about:blank" || iframeUrl.startsWith("javascript:")) continue;
+      const frameEls = await domFallbackElements(frame).catch(() => []);
+      for (const el of frameEls.slice(0, 10)) {
+        iframeElements.push({ ...el, notes: el.notes ? `${el.notes} [in iframe]` : "[in iframe]" });
+      }
+    }
+  } catch {
+  }
+  let dynamicElements = [];
+  if (domElements.length > 0) {
+    try {
+      dynamicElements = await page.evaluate(() => {
+        const out = [];
+        const seen = /* @__PURE__ */ new Set();
+        const triggers = document.querySelectorAll('[role="combobox"],[role="listbox"] [role="option"],[aria-haspopup]');
+        triggers.forEach((el) => {
+          const label = (el.getAttribute("aria-label") || el.textContent?.trim() || "").slice(0, 80);
+          if (!label || seen.has(label)) return;
+          const role = el.getAttribute("role") ?? el.tagName.toLowerCase();
+          seen.add(label);
+          out.push({ meaning: label, role, notes: "[dynamic/revealed]" });
+        });
+        return out.slice(0, 10);
+      }).catch(() => []);
+    } catch {
+      dynamicElements = [];
+    }
+  }
+  if (domElements.length >= 2) return [...domElements, ...iframeElements, ...dynamicElements];
   const contentElements = await contentFallbackElements(page);
   if (contentElements.length > 0) return [...domElements, ...contentElements];
   try {
@@ -3540,8 +3673,6 @@ async function waitForScreenshotReady(page) {
     }).catch(() => {
     }));
     await withHardCap(waitForStableDOM(page, 2e3, 500));
-    await withHardCap(page.waitForTimeout(250).catch(() => {
-    }));
   } catch {
   }
 }