@detiq/crawler 1.1.15 → 1.1.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.15",
+  "version": "1.1.16",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -1288,6 +1288,17 @@ export async function crawlProject(opts: {
             logger.info({ url: finalUrl }, '[crawler] pre-auth page has 0 elements — deferring to authenticated BFS');
             continue;
           }
+          // Detect auth walls that have a tiny number of elements (login button, back link, spinner).
+          // These pages redirect or gate on auth — main BFS must visit them with a live session.
+          // Label prefix or exact match — "Back to Learn" matches "back to", "Sign In" matches "sign in"
+          const AUTH_WALL_LABEL_RE = /^(sign.?in|log.?in|login|platform access|back\s+to|checking\s+auth|get started|create account|register|continue with|join now)/i;
+          // Also treat pages with only `a`-role elements (no inputs/buttons) as likely auth walls
+          const allAnchorOnly = preAuthElements.every(el => el.role === 'a');
+          const looksLikeAuthWall = preAuthElements.length <= 3 && (allAnchorOnly || preAuthElements.every(el => AUTH_WALL_LABEL_RE.test(el.label)));
+          if (looksLikeAuthWall) {
+            logger.info({ url: finalUrl, elements: preAuthElements.length, labels: preAuthElements.map(e => e.label) }, '[crawler] pre-auth auth wall — deferring to authenticated BFS');
+            continue;
+          }
           const pathParts = finalPath.split('/').filter(Boolean);
           const screenName = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, finalUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
@@ -2026,7 +2037,16 @@ export async function crawlProject(opts: {
               const s = window.getComputedStyle(el);
               return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
             }
-            document.querySelectorAll('nav a, [role="navigation"] a, header a, footer nav a, [aria-label*="navigation" i] a').forEach((a: any) => {
+            // Wide nav selector: covers semantic nav, ARIA, header links, sidebar (aside/[role=complementary]),
+            // and common class-based sidebars used by admin/dashboard pages.
+            const NAV_SEL = [
+              'nav a', '[role="navigation"] a', 'header a', 'footer nav a',
+              '[aria-label*="navigation" i] a', '[aria-label*="sidebar" i] a',
+              'aside a', '[role="complementary"] a',
+              '[class*="sidebar" i] a', '[class*="side-nav" i] a', '[class*="sidenav" i] a',
+              '[class*="nav-menu" i] a', '[class*="navmenu" i] a', '[data-nav] a',
+            ].join(', ');
+            document.querySelectorAll(NAV_SEL).forEach((a: any) => {
               if (!isV(a)) return;
               const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
               const href = a.href ?? '';
@@ -2036,8 +2056,10 @@ export async function crawlProject(opts: {
               while (cur) {
                 const t = cur.tagName?.toLowerCase();
                 const r = cur.getAttribute?.('role')?.toLowerCase();
+                const cl = (cur.className ?? '').toLowerCase();
                 if (t === 'header' || r === 'banner') { landmark = 'Header'; break; }
                 if (t === 'footer' || r === 'contentinfo') { landmark = 'Footer'; break; }
+                if (t === 'aside' || r === 'complementary' || cl.includes('sidebar') || cl.includes('sidenav') || cl.includes('side-nav')) { landmark = 'Sidebar'; break; }
                 if (t === 'nav' || r === 'navigation') { landmark = cur.getAttribute('aria-label')?.trim() || 'Site Navigation'; break; }
                 cur = cur.parentElement;
               }
@@ -2053,6 +2075,42 @@ export async function crawlProject(opts: {
               const key = `footer|${label.toLowerCase()}|${href}`;
               if (!seen.has(key)) { seen.add(key); items.push({ meaning: label, role: 'footer-link', notes: JSON.stringify({ href: href.slice(0, 150) }) }); fc++; }
             });
+            // Grandparent-grouping fallback: detects sidebar navigation where each link is
+            // individually wrapped (e.g. <div class="flex"><a>link</a></div> × many siblings).
+            // Runs ALWAYS (not gated on 0 results) so header links don't block sidebar detection.
+            // Deduped against hrefs already captured by the semantic pass above.
+            {
+              const navHrefsSeen = new Set<string>();
+              for (const it of items) {
+                try { const h = JSON.parse(it.notes ?? '{}').href; if (h) navHrefsSeen.add(h); } catch { /* skip */ }
+              }
+              const gpGroups = new Map<Element, Array<{label: string; href: string}>>();
+              document.querySelectorAll('a[href]').forEach((a: any) => {
+                if (!isV(a)) return;
+                const href = a.href ?? '';
+                if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsSeen.has(href)) return;
+                const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+                if (!label || label.length > 40) return;
+                const parent = a.parentElement;
+                const gp = parent?.parentElement;
+                if (!parent || !gp) return;
+                // Only match when parent wraps exactly 1 anchor — the sidebar <div><a> pattern
+                if (parent.querySelectorAll('a').length !== 1) return;
+                const gpTag = gp.tagName.toLowerCase();
+                if (['body', 'html', 'form'].includes(gpTag)) return;
+                const arr = gpGroups.get(gp) ?? [];
+                arr.push({ label, href });
+                gpGroups.set(gp, arr);
+              });
+              for (const [, links] of gpGroups.entries()) {
+                if (links.length >= 4) {
+                  for (const { label, href } of links) {
+                    const k = `cluster|${label.toLowerCase()}|${href}`;
+                    if (!seen.has(k)) { seen.add(k); items.push({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark: 'Sidebar' }) }); }
+                  }
+                }
+              }
+            }
             return items;
           }).catch(() => [] as any[]);
 
@@ -2807,7 +2865,7 @@ export async function crawlProject(opts: {
             }
 
             // Append significant query params
-            const SIG_PARAMS = ['product_cat', 'category', 'cat', 'type', 'tag', 'brand', 'filter', 'tab', 'section', 'view'];
+            const SIG_PARAMS = ['product_cat', 'category', 'cat', 'type', 'tag', 'brand', 'filter', 'tab', 'section', 'view', 'play', 'mode', 'id', 'page'];
             const paramSuffix = SIG_PARAMS
               .map((p) => { const v = u.searchParams.get(p); return v ? ` · ${v.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())}` : ''; })
               .find(Boolean) ?? '';
@@ -2830,11 +2888,12 @@ export async function crawlProject(opts: {
                 const candidate = parts[0].trim();
                 if (candidate.length > 2 && !/^\d[\d\s]*$/.test(candidate)) return candidate;
               }
-              // Single-segment title — only use if multi-word and not brand-name-like
+              // Single-segment title — only use if multi-word, not brand-name-like, and not the generic SPA app title
               const candidate = parts[0].trim();
               const isBrand = candidate.split(/\s+/).length === 1 &&
                 (candidate.toLowerCase().includes(brandName) || candidate.toLowerCase() === brandName);
-              if (!isBrand && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
+              const isSiteGenericTitle = homeFingerprintTitle !== '' && candidate === homeFingerprintTitle;
+              if (!isBrand && !isSiteGenericTitle && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
                 return candidate;
               }
             }
@@ -3365,10 +3424,38 @@ export async function crawlProject(opts: {
             try {
               const baseline = await AppBrain.getVisualBaselineForDiff(opts.tenantId, screen.id, 'DESKTOP', gitInfo.branch);
               if (baseline && baseline.storagePath !== shotPath) {
+                // Resolve baseline file path — local copy may have been rotated; fall back to cloud download
+                let baselinePath: string = baseline.storagePath;
+                let tempBaselinePath: string | null = null;
+                const localExists = await fs.access(baseline.storagePath).then(() => true, () => false);
+                if (!localExists) {
+                  if (baseline.cloudUrl) {
+                    try {
+                      const resp = await fetch(baseline.cloudUrl);
+                      if (resp.ok) {
+                        const buf = Buffer.from(await resp.arrayBuffer());
+                        tempBaselinePath = `${shotPath}.baseline-tmp.png`;
+                        await fs.writeFile(tempBaselinePath, buf);
+                        baselinePath = tempBaselinePath;
+                        logger.info({ screenId: screen.id }, '[crawler] VisualDiff baseline downloaded from cloud storage');
+                      } else {
+                        logger.warn({ screenId: screen.id, status: resp.status }, '[crawler] VisualDiff skipped — baseline cloud fetch returned non-200');
+                        return;
+                      }
+                    } catch (dlErr) {
+                      logger.warn({ screenId: screen.id, err: String(dlErr) }, '[crawler] VisualDiff skipped — baseline cloud download failed');
+                      return;
+                    }
+                  } else {
+                    logger.warn({ screenId: screen.id }, '[crawler] VisualDiff skipped — baseline not on disk and no cloudUrl available');
+                    return;
+                  }
+                }
+                try {
                 const { screenshotDiffPercent, gridLayoutDiff } = await import('./phash.js');
                 const [{ pHashDistance, changeLevel }, layoutChanges, elementShifts] = await Promise.all([
-                  screenshotDiffPercent(baseline.storagePath, shotPath, baseline.ignoredRegions),
-                  gridLayoutDiff(baseline.storagePath, shotPath, baseline.ignoredRegions),
+                  screenshotDiffPercent(baselinePath, shotPath, baseline.ignoredRegions),
+                  gridLayoutDiff(baselinePath, shotPath, baseline.ignoredRegions),
                   AppBrain.computeElementLayoutDiff(opts.tenantId, screen.id, 'DESKTOP'),
                 ]);
                 let verdict = pHashVerdict(pHashDistance, changeLevel, [
@@ -3389,10 +3476,10 @@ export async function crawlProject(opts: {
                     const [baselineBuffer, newBuffer] = regions.length > 0
                       ? await (async () => {
                         const { applyMask } = await import('./phash.js');
-                        return Promise.all([applyMask(baseline.storagePath, regions), applyMask(shotPath, regions)]);
+                        return Promise.all([applyMask(baselinePath, regions), applyMask(shotPath, regions)]);
                       })()
                       : await Promise.all([
-                        fs.readFile(baseline.storagePath),
+                        fs.readFile(baselinePath),
                         fs.readFile(shotPath),
                       ]);
                     const aiResult = await aiVisualCompare({
@@ -3435,7 +3522,7 @@ export async function crawlProject(opts: {
                       // R5: Generate and upload pixelmatch diff image (changed pixels in red)
                       try {
                         const { generatePixelmatchDiff } = await import('./phash.js');
-                        const diffBuffer = await generatePixelmatchDiff(baseline.storagePath, shotPath, baseline.ignoredRegions ?? []);
+                        const diffBuffer = await generatePixelmatchDiff(baselinePath, shotPath, baseline.ignoredRegions ?? []);
                         const diffTmpPath = `${shotPath}.diff.png`;
                         await fs.writeFile(diffTmpPath, diffBuffer);
                         const diffKey = buildArtifactKey(opts.tenantId, opts.projectId, 'visual-diffs', `${screen.id}-v${screen.version}-diff.png`, storageCfg.prefix);
@@ -3459,14 +3546,12 @@ export async function crawlProject(opts: {
                     aiSummary,
                   });
                 }
+                } finally {
+                  if (tempBaselinePath) fs.unlink(tempBaselinePath).catch(() => {});
+                }
               }
             } catch (err: any) {
-              const missing = String(err?.message ?? err).includes('Input file is missing');
-              if (missing) {
-                logger.warn({ screenId: screen.id, version: screen.version, err: String(err?.message ?? err) }, '[crawler] VisualDiff skipped — baseline screenshot not on disk (stored in cloud only). Run a full recrawl to regenerate baselines.');
-              } else {
-                console.warn('[crawler] VisualDiff pipeline failed:', err);
-              }
+              console.warn('[crawler] VisualDiff pipeline failed:', err);
             }
           })();
         }
@@ -5094,19 +5179,28 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
       const s = window.getComputedStyle(el);
       return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
     };
-    document.querySelectorAll('nav a, [role="navigation"] a, header a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a: any) => {
+    const navSel = [
+      'nav a', '[role="navigation"] a', 'header a', 'header nav a', 'footer nav a',
+      '[aria-label*="navigation" i] a', '[aria-label*="menu" i] a', '[aria-label*="sidebar" i] a',
+      'aside a', '[role="complementary"] a',
+      '[class*="sidebar" i] a', '[class*="side-nav" i] a', '[class*="sidenav" i] a',
+      '[class*="nav-menu" i] a', '[class*="navmenu" i] a', '[data-nav] a',
+    ].join(', ');
+    const navLinksFoundCount = { n: 0 };
+    document.querySelectorAll(navSel).forEach((a: any) => {
       if (!isNavVisible(a)) return;
       const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
       const href = a.href;
       if (!label || !href || /^(javascript:|#$)/i.test(href)) return;
-      // Walk up to find nearest nav/header/footer ancestor to get landmark label
       let landmark = 'Site Navigation';
       let el: any = a.parentElement;
       while (el) {
         const tag = el.tagName?.toLowerCase();
         const role = el.getAttribute?.('role')?.toLowerCase();
+        const cl = (el.className ?? '').toLowerCase();
         if (tag === 'header' || role === 'banner') { landmark = 'Header'; break; }
         if (tag === 'footer' || role === 'contentinfo') { landmark = 'Footer'; break; }
+        if (tag === 'aside' || role === 'complementary' || cl.includes('sidebar') || cl.includes('sidenav') || cl.includes('side-nav')) { landmark = 'Sidebar'; break; }
         if (tag === 'nav' || role === 'navigation') {
           const directLabel = el.getAttribute('aria-label')?.trim() ?? '';
           const labelId = el.getAttribute('aria-labelledby');
@@ -5117,7 +5211,43 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
         el = el.parentElement;
       }
       addUniq({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark }) });
+      navLinksFoundCount.n++;
     });
+    // Grandparent-grouping fallback: detects sidebar navigation where each link is individually
+    // wrapped (e.g. <div class="flex"><a>link</a></div> × many siblings). Runs ALWAYS so header
+    // links don't block sidebar detection. Deduped against hrefs already in the semantic pass.
+    {
+      const navHrefsAlready = new Set<string>();
+      for (const item of out) {
+        if (item.role === 'nav-link') {
+          try { const h = JSON.parse(item.notes ?? '{}').href; if (h) navHrefsAlready.add(h); } catch { /* skip */ }
+        }
+      }
+      const gpGroups2 = new Map<Element, Array<{label: string; href: string}>>();
+      document.querySelectorAll('a[href]').forEach((a: any) => {
+        if (!isNavVisible(a)) return;
+        const href = a.href ?? '';
+        if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsAlready.has(href)) return;
+        const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+        if (!label || label.length > 40) return;
+        const parent = a.parentElement;
+        const gp = parent?.parentElement;
+        if (!parent || !gp) return;
+        if (parent.querySelectorAll('a').length !== 1) return;
+        const gpTag = gp.tagName.toLowerCase();
+        if (['body', 'html', 'form'].includes(gpTag)) return;
+        const arr = gpGroups2.get(gp) ?? [];
+        arr.push({ label, href });
+        gpGroups2.set(gp, arr);
+      });
+      for (const [, links] of gpGroups2.entries()) {
+        if (links.length >= 4) {
+          for (const { label, href } of links) {
+            addUniq({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark: 'Sidebar' }) });
+          }
+        }
+      }
+    }
 
     // 9. Breadcrumb items
     document.querySelectorAll('[aria-label*="breadcrumb" i] a, [class*="breadcrumb" i] a, nav[aria-label*="breadcrumb" i] a').forEach((a: any) => {

package/src/job-queue.ts

@@ -2502,6 +2502,15 @@ jobDispatchTable = {
   BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
   CREATE_PR: processCreatePRJob,
   NEW_PAGES_PIPELINE: processNewPagesPipelineJob,
+  // GENERATE_TESTS is processed in-process by the API (never QUEUED).
+  // If the worker claims one, the API crashed after creating it — mark failed so it's visible.
+  GENERATE_TESTS: async (jobId: string, _config: any) => {
+    logger.error({ jobId }, '[job-worker] GENERATE_TESTS claimed by worker — API did not process it in-process');
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { status: 'FAILED', error: 'Test generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
+    }).catch(() => {});
+  },
 };
 
 async function processRepoScanJob(jobId: string, config: any) {