@detiq/crawler 1.0.7 → 1.0.9

package/dist/crawler.js

@@ -204,7 +204,8 @@ async function launchChromeCDP(execPath, extraArgs, proxy) {
       `--remote-debugging-port=${port}`,
       "--remote-allow-origins=*",
       "--no-first-run",
-      "--no-default-browser-check"
+      "--no-default-browser-check",
+      "--window-size=1280,900"
     ]
   });
   const deadline = Date.now() + 1e4;
@@ -2183,6 +2184,8 @@ ${snippet}`,
           })();
         }
         const linkMappings = await extractLinkMappings(page, origin);
+        await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
+        });
         for (let _ei = 0; _ei < elements.length; _ei += 4) {
           const _batch = elements.slice(_ei, _ei + 4);
           await Promise.all(_batch.map(async (el) => {
@@ -3393,8 +3396,12 @@ async function contentFallbackElements(page) {
       const text = p.textContent?.replace(/\s+/g, " ").trim().slice(0, 300);
       if (text && text.length > 10 && text.length < 400) addUniq({ meaning: text, role: "cta-section" });
     });
-    document.querySelectorAll('nav a, [role="navigation"] a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a) => {
-      if (!isVisible(a)) return;
+    const isNavVisible = (el) => {
+      const s = window.getComputedStyle(el);
+      return s.display !== "none" && s.visibility !== "hidden" && s.opacity !== "0";
+    };
+    document.querySelectorAll('nav a, [role="navigation"] a, header a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a) => {
+      if (!isNavVisible(a)) return;
       const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 60);
       const href = a.href;
       if (!label || !href || /^(javascript:|#$)/i.test(href)) return;