@detiq/crawler 1.1.15 → 1.1.16

package/dist/crawler.js

@@ -951,6 +951,13 @@ async function crawlProject(opts, onProgress) {
             logger.info({ url: finalUrl }, "[crawler] pre-auth page has 0 elements \u2014 deferring to authenticated BFS");
             continue;
           }
+          const AUTH_WALL_LABEL_RE = /^(sign.?in|log.?in|login|platform access|back\s+to|checking\s+auth|get started|create account|register|continue with|join now)/i;
+          const allAnchorOnly = preAuthElements.every((el) => el.role === "a");
+          const looksLikeAuthWall = preAuthElements.length <= 3 && (allAnchorOnly || preAuthElements.every((el) => AUTH_WALL_LABEL_RE.test(el.label)));
+          if (looksLikeAuthWall) {
+            logger.info({ url: finalUrl, elements: preAuthElements.length, labels: preAuthElements.map((e) => e.label) }, "[crawler] pre-auth auth wall \u2014 deferring to authenticated BFS");
+            continue;
+          }
           const pathParts = finalPath.split("/").filter(Boolean);
           const screenName = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, finalUrl, opts.jobId, "LINK_FOLLOW").catch(() => null);
@@ -1644,7 +1651,23 @@ async function crawlProject(opts, onProgress) {
               const s = window.getComputedStyle(el);
               return s.display !== "none" && s.visibility !== "hidden" && s.opacity !== "0";
             }
-            document.querySelectorAll('nav a, [role="navigation"] a, header a, footer nav a, [aria-label*="navigation" i] a').forEach((a) => {
+            const NAV_SEL = [
+              "nav a",
+              '[role="navigation"] a',
+              "header a",
+              "footer nav a",
+              '[aria-label*="navigation" i] a',
+              '[aria-label*="sidebar" i] a',
+              "aside a",
+              '[role="complementary"] a',
+              '[class*="sidebar" i] a',
+              '[class*="side-nav" i] a',
+              '[class*="sidenav" i] a',
+              '[class*="nav-menu" i] a',
+              '[class*="navmenu" i] a',
+              "[data-nav] a"
+            ].join(", ");
+            document.querySelectorAll(NAV_SEL).forEach((a) => {
               if (!isV(a)) return;
               const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 60);
               const href = a.href ?? "";
@@ -1654,6 +1677,7 @@ async function crawlProject(opts, onProgress) {
               while (cur) {
                 const t = cur.tagName?.toLowerCase();
                 const r = cur.getAttribute?.("role")?.toLowerCase();
+                const cl = (cur.className ?? "").toLowerCase();
                 if (t === "header" || r === "banner") {
                   landmark = "Header";
                   break;
@@ -1662,6 +1686,10 @@ async function crawlProject(opts, onProgress) {
                   landmark = "Footer";
                   break;
                 }
+                if (t === "aside" || r === "complementary" || cl.includes("sidebar") || cl.includes("sidenav") || cl.includes("side-nav")) {
+                  landmark = "Sidebar";
+                  break;
+                }
                 if (t === "nav" || r === "navigation") {
                   landmark = cur.getAttribute("aria-label")?.trim() || "Site Navigation";
                   break;
@@ -1687,6 +1715,44 @@ async function crawlProject(opts, onProgress) {
                 fc++;
               }
             });
+            {
+              const navHrefsSeen = /* @__PURE__ */ new Set();
+              for (const it of items) {
+                try {
+                  const h = JSON.parse(it.notes ?? "{}").href;
+                  if (h) navHrefsSeen.add(h);
+                } catch {
+                }
+              }
+              const gpGroups = /* @__PURE__ */ new Map();
+              document.querySelectorAll("a[href]").forEach((a) => {
+                if (!isV(a)) return;
+                const href = a.href ?? "";
+                if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsSeen.has(href)) return;
+                const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 60);
+                if (!label || label.length > 40) return;
+                const parent = a.parentElement;
+                const gp = parent?.parentElement;
+                if (!parent || !gp) return;
+                if (parent.querySelectorAll("a").length !== 1) return;
+                const gpTag = gp.tagName.toLowerCase();
+                if (["body", "html", "form"].includes(gpTag)) return;
+                const arr = gpGroups.get(gp) ?? [];
+                arr.push({ label, href });
+                gpGroups.set(gp, arr);
+              });
+              for (const [, links] of gpGroups.entries()) {
+                if (links.length >= 4) {
+                  for (const { label, href } of links) {
+                    const k = `cluster|${label.toLowerCase()}|${href}`;
+                    if (!seen.has(k)) {
+                      seen.add(k);
+                      items.push({ meaning: label, role: "nav-link", notes: JSON.stringify({ href: href.slice(0, 150), landmark: "Sidebar" }) });
+                    }
+                  }
+                }
+              }
+            }
             return items;
           }).catch(() => []);
           const allElements = [
@@ -2329,7 +2395,7 @@ async function crawlProject(opts, onProgress) {
               const lastSeg = allSegs[allSegs.length - 1];
               pathName = lastSeg.slice(0, 8).toUpperCase();
             }
-            const SIG_PARAMS = ["product_cat", "category", "cat", "type", "tag", "brand", "filter", "tab", "section", "view"];
+            const SIG_PARAMS = ["product_cat", "category", "cat", "type", "tag", "brand", "filter", "tab", "section", "view", "play", "mode", "id", "page"];
             const paramSuffix = SIG_PARAMS.map((p) => {
               const v = u.searchParams.get(p);
               return v ? ` \xB7 ${v.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase())}` : "";
@@ -2349,7 +2415,8 @@ async function crawlProject(opts, onProgress) {
               }
               const candidate = parts[0].trim();
               const isBrand = candidate.split(/\s+/).length === 1 && (candidate.toLowerCase().includes(brandName) || candidate.toLowerCase() === brandName);
-              if (!isBrand && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
+              const isSiteGenericTitle = homeFingerprintTitle !== "" && candidate === homeFingerprintTitle;
+              if (!isBrand && !isSiteGenericTitle && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
                 return candidate;
               }
             }
@@ -2843,95 +2910,121 @@ ${snippet}`,
             try {
               const baseline = await AppBrain.getVisualBaselineForDiff(opts.tenantId, screen.id, "DESKTOP", gitInfo.branch);
               if (baseline && baseline.storagePath !== shotPath) {
-                const { screenshotDiffPercent, gridLayoutDiff } = await import("./phash.js");
-                const [{ pHashDistance, changeLevel }, layoutChanges, elementShifts] = await Promise.all([
-                  screenshotDiffPercent(baseline.storagePath, shotPath, baseline.ignoredRegions),
-                  gridLayoutDiff(baseline.storagePath, shotPath, baseline.ignoredRegions),
-                  AppBrain.computeElementLayoutDiff(opts.tenantId, screen.id, "DESKTOP")
-                ]);
-                let verdict = pHashVerdict(pHashDistance, changeLevel, [
-                  ...layoutChanges,
-                  ...elementShifts.map((s) => ({ location: "element", severity: s.severity, description: s.description }))
-                ]);
-                if (verdict.changeType === "SIGNIFICANT" || verdict.changeType === "CRITICAL") {
-                  try {
-                    const regions = baseline.ignoredRegions ?? [];
-                    const [baselineBuffer, newBuffer] = regions.length > 0 ? await (async () => {
-                      const { applyMask } = await import("./phash.js");
-                      return Promise.all([applyMask(baseline.storagePath, regions), applyMask(shotPath, regions)]);
-                    })() : await Promise.all([
-                      fs.readFile(baseline.storagePath),
-                      fs.readFile(shotPath)
-                    ]);
-                    const aiResult = await aiVisualCompare({
-                      baselineImageBase64: baselineBuffer.toString("base64"),
-                      newImageBase64: newBuffer.toString("base64"),
-                      screenName: screen.name,
-                      screenPurpose: "Application screen captured during automated crawl",
-                      tenantId: opts.tenantId,
-                      projectId: opts.projectId
-                    });
-                    verdict = mergeAiVisualVerdict(verdict, aiResult, screen.name);
-                  } catch (aiErr) {
-                    logger.warn({ screenId: screen.id, err: String(aiErr) }, "[crawler] AI visual compare failed, keeping pHash-only verdict");
+                let baselinePath = baseline.storagePath;
+                let tempBaselinePath = null;
+                const localExists = await fs.access(baseline.storagePath).then(() => true, () => false);
+                if (!localExists) {
+                  if (baseline.cloudUrl) {
+                    try {
+                      const resp = await fetch(baseline.cloudUrl);
+                      if (resp.ok) {
+                        const buf = Buffer.from(await resp.arrayBuffer());
+                        tempBaselinePath = `${shotPath}.baseline-tmp.png`;
+                        await fs.writeFile(tempBaselinePath, buf);
+                        baselinePath = tempBaselinePath;
+                        logger.info({ screenId: screen.id }, "[crawler] VisualDiff baseline downloaded from cloud storage");
+                      } else {
+                        logger.warn({ screenId: screen.id, status: resp.status }, "[crawler] VisualDiff skipped \u2014 baseline cloud fetch returned non-200");
+                        return;
+                      }
+                    } catch (dlErr) {
+                      logger.warn({ screenId: screen.id, err: String(dlErr) }, "[crawler] VisualDiff skipped \u2014 baseline cloud download failed");
+                      return;
+                    }
+                  } else {
+                    logger.warn({ screenId: screen.id }, "[crawler] VisualDiff skipped \u2014 baseline not on disk and no cloudUrl available");
+                    return;
                   }
                 }
-                let { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
-                if (changeType !== "NONE") {
-                  try {
-                    const project = await AppBrain.getProject(opts.tenantId, opts.projectId);
-                    const threshold = project?.visualDiffThreshold ?? null;
-                    if (threshold !== null && pHashDistance < threshold) {
-                      changeType = "NONE";
+                try {
+                  const { screenshotDiffPercent, gridLayoutDiff } = await import("./phash.js");
+                  const [{ pHashDistance, changeLevel }, layoutChanges, elementShifts] = await Promise.all([
+                    screenshotDiffPercent(baselinePath, shotPath, baseline.ignoredRegions),
+                    gridLayoutDiff(baselinePath, shotPath, baseline.ignoredRegions),
+                    AppBrain.computeElementLayoutDiff(opts.tenantId, screen.id, "DESKTOP")
+                  ]);
+                  let verdict = pHashVerdict(pHashDistance, changeLevel, [
+                    ...layoutChanges,
+                    ...elementShifts.map((s) => ({ location: "element", severity: s.severity, description: s.description }))
+                  ]);
+                  if (verdict.changeType === "SIGNIFICANT" || verdict.changeType === "CRITICAL") {
+                    try {
+                      const regions = baseline.ignoredRegions ?? [];
+                      const [baselineBuffer, newBuffer] = regions.length > 0 ? await (async () => {
+                        const { applyMask } = await import("./phash.js");
+                        return Promise.all([applyMask(baselinePath, regions), applyMask(shotPath, regions)]);
+                      })() : await Promise.all([
+                        fs.readFile(baselinePath),
+                        fs.readFile(shotPath)
+                      ]);
+                      const aiResult = await aiVisualCompare({
+                        baselineImageBase64: baselineBuffer.toString("base64"),
+                        newImageBase64: newBuffer.toString("base64"),
+                        screenName: screen.name,
+                        screenPurpose: "Application screen captured during automated crawl",
+                        tenantId: opts.tenantId,
+                        projectId: opts.projectId
+                      });
+                      verdict = mergeAiVisualVerdict(verdict, aiResult, screen.name);
+                    } catch (aiErr) {
+                      logger.warn({ screenId: screen.id, err: String(aiErr) }, "[crawler] AI visual compare failed, keeping pHash-only verdict");
                     }
-                  } catch {
                   }
-                }
-                if (changeType !== "NONE") {
-                  let newScreenshotCloudUrl;
-                  let diffImageCloudUrl;
-                  try {
-                    const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import("@detiq/app-brain");
-                    const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
-                    if (storageCfg) {
-                      const key = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
-                      newScreenshotCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, shotPath, key);
-                      try {
-                        const { generatePixelmatchDiff } = await import("./phash.js");
-                        const diffBuffer = await generatePixelmatchDiff(baseline.storagePath, shotPath, baseline.ignoredRegions ?? []);
-                        const diffTmpPath = `${shotPath}.diff.png`;
-                        await fs.writeFile(diffTmpPath, diffBuffer);
-                        const diffKey = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}-diff.png`, storageCfg.prefix);
-                        diffImageCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, diffTmpPath, diffKey);
-                        fs.unlink(diffTmpPath).catch(() => {
-                        });
-                      } catch (diffErr) {
-                        logger.warn({ screenId: screen.id, err: String(diffErr) }, "[crawler] pixelmatch diff generation failed, skipping");
+                  let { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
+                  if (changeType !== "NONE") {
+                    try {
+                      const project = await AppBrain.getProject(opts.tenantId, opts.projectId);
+                      const threshold = project?.visualDiffThreshold ?? null;
+                      if (threshold !== null && pHashDistance < threshold) {
+                        changeType = "NONE";
                       }
+                    } catch {
                     }
-                  } catch {
                   }
-                  await AppBrain.saveVisualDiff(opts.tenantId, {
-                    baselineId: baseline.id,
-                    newScreenshotPath: shotPath,
-                    newScreenshotCloudUrl,
-                    diffImageCloudUrl,
-                    diffScore: pHashDistance,
-                    changeType,
-                    changes: allChanges,
-                    recommendation,
-                    bugReportDraft,
-                    aiSummary
+                  if (changeType !== "NONE") {
+                    let newScreenshotCloudUrl;
+                    let diffImageCloudUrl;
+                    try {
+                      const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import("@detiq/app-brain");
+                      const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
+                      if (storageCfg) {
+                        const key = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
+                        newScreenshotCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, shotPath, key);
+                        try {
+                          const { generatePixelmatchDiff } = await import("./phash.js");
+                          const diffBuffer = await generatePixelmatchDiff(baselinePath, shotPath, baseline.ignoredRegions ?? []);
+                          const diffTmpPath = `${shotPath}.diff.png`;
+                          await fs.writeFile(diffTmpPath, diffBuffer);
+                          const diffKey = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}-diff.png`, storageCfg.prefix);
+                          diffImageCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, diffTmpPath, diffKey);
+                          fs.unlink(diffTmpPath).catch(() => {
+                          });
+                        } catch (diffErr) {
+                          logger.warn({ screenId: screen.id, err: String(diffErr) }, "[crawler] pixelmatch diff generation failed, skipping");
+                        }
+                      }
+                    } catch {
+                    }
+                    await AppBrain.saveVisualDiff(opts.tenantId, {
+                      baselineId: baseline.id,
+                      newScreenshotPath: shotPath,
+                      newScreenshotCloudUrl,
+                      diffImageCloudUrl,
+                      diffScore: pHashDistance,
+                      changeType,
+                      changes: allChanges,
+                      recommendation,
+                      bugReportDraft,
+                      aiSummary
+                    });
+                  }
+                } finally {
+                  if (tempBaselinePath) fs.unlink(tempBaselinePath).catch(() => {
                   });
                 }
               }
             } catch (err) {
-              const missing = String(err?.message ?? err).includes("Input file is missing");
-              if (missing) {
-                logger.warn({ screenId: screen.id, version: screen.version, err: String(err?.message ?? err) }, "[crawler] VisualDiff skipped \u2014 baseline screenshot not on disk (stored in cloud only). Run a full recrawl to regenerate baselines.");
-              } else {
-                console.warn("[crawler] VisualDiff pipeline failed:", err);
-              }
+              console.warn("[crawler] VisualDiff pipeline failed:", err);
             }
           })();
         }
@@ -4355,7 +4448,26 @@ async function contentFallbackElements(page) {
       const s = window.getComputedStyle(el);
       return s.display !== "none" && s.visibility !== "hidden" && s.opacity !== "0";
     };
-    document.querySelectorAll('nav a, [role="navigation"] a, header a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a) => {
+    const navSel = [
+      "nav a",
+      '[role="navigation"] a',
+      "header a",
+      "header nav a",
+      "footer nav a",
+      '[aria-label*="navigation" i] a',
+      '[aria-label*="menu" i] a',
+      '[aria-label*="sidebar" i] a',
+      "aside a",
+      '[role="complementary"] a',
+      '[class*="sidebar" i] a',
+      '[class*="side-nav" i] a',
+      '[class*="sidenav" i] a',
+      '[class*="nav-menu" i] a',
+      '[class*="navmenu" i] a',
+      "[data-nav] a"
+    ].join(", ");
+    const navLinksFoundCount = { n: 0 };
+    document.querySelectorAll(navSel).forEach((a) => {
       if (!isNavVisible(a)) return;
       const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 60);
       const href = a.href;
@@ -4365,6 +4477,7 @@ async function contentFallbackElements(page) {
       while (el) {
         const tag = el.tagName?.toLowerCase();
         const role = el.getAttribute?.("role")?.toLowerCase();
+        const cl = (el.className ?? "").toLowerCase();
         if (tag === "header" || role === "banner") {
           landmark = "Header";
           break;
@@ -4373,6 +4486,10 @@ async function contentFallbackElements(page) {
           landmark = "Footer";
           break;
         }
+        if (tag === "aside" || role === "complementary" || cl.includes("sidebar") || cl.includes("sidenav") || cl.includes("side-nav")) {
+          landmark = "Sidebar";
+          break;
+        }
         if (tag === "nav" || role === "navigation") {
           const directLabel = el.getAttribute("aria-label")?.trim() ?? "";
           const labelId = el.getAttribute("aria-labelledby");
@@ -4383,7 +4500,44 @@ async function contentFallbackElements(page) {
         el = el.parentElement;
       }
       addUniq({ meaning: label, role: "nav-link", notes: JSON.stringify({ href: href.slice(0, 150), landmark }) });
+      navLinksFoundCount.n++;
     });
+    {
+      const navHrefsAlready = /* @__PURE__ */ new Set();
+      for (const item of out) {
+        if (item.role === "nav-link") {
+          try {
+            const h = JSON.parse(item.notes ?? "{}").href;
+            if (h) navHrefsAlready.add(h);
+          } catch {
+          }
+        }
+      }
+      const gpGroups2 = /* @__PURE__ */ new Map();
+      document.querySelectorAll("a[href]").forEach((a) => {
+        if (!isNavVisible(a)) return;
+        const href = a.href ?? "";
+        if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsAlready.has(href)) return;
+        const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 60);
+        if (!label || label.length > 40) return;
+        const parent = a.parentElement;
+        const gp = parent?.parentElement;
+        if (!parent || !gp) return;
+        if (parent.querySelectorAll("a").length !== 1) return;
+        const gpTag = gp.tagName.toLowerCase();
+        if (["body", "html", "form"].includes(gpTag)) return;
+        const arr = gpGroups2.get(gp) ?? [];
+        arr.push({ label, href });
+        gpGroups2.set(gp, arr);
+      });
+      for (const [, links] of gpGroups2.entries()) {
+        if (links.length >= 4) {
+          for (const { label, href } of links) {
+            addUniq({ meaning: label, role: "nav-link", notes: JSON.stringify({ href: href.slice(0, 150), landmark: "Sidebar" }) });
+          }
+        }
+      }
+    }
     document.querySelectorAll('[aria-label*="breadcrumb" i] a, [class*="breadcrumb" i] a, nav[aria-label*="breadcrumb" i] a').forEach((a) => {
       const label = a.textContent?.replace(/\s+/g, " ").trim().slice(0, 60);
       const href = a.href;