@detiq/crawler 1.1.5 → 1.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.5",
+  "version": "1.1.6",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -28,7 +28,7 @@ import { z } from 'zod';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { AppBrain, recordLLMCall } from '@detiq/app-brain';
-import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
+import { aiVisualCompare, askTracked, deterministicSeed, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
 import { config } from '@detiq/core';
 import { pHashVerdict, mergeAiVisualVerdict } from './visual-diff-verdict.js';
 import type { Credentials } from './auth.js';
@@ -1125,6 +1125,14 @@ export async function crawlProject(opts: {
     await page.waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
     await waitForStableDOM(page);
 
+    // Capture homepage fingerprint for soft-404 detection. SPA catch-all routes return HTTP 200
+    // but render identical homepage content for every unrecognised path. We fingerprint here
+    // (pre-auth, stable DOM) and skip any BFS page that matches during the main crawl.
+    const homeFingerprintTitle = await page.title().catch(() => '');
+    const homeFingerprintElemCount = await page.evaluate(() =>
+      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
+
     // Dedicated logged-out/auth discovery phase. This runs before credentials or
     // storageState are applied, so public pages and auth recovery flows remain
     // visible even when the main crawl later becomes authenticated.
@@ -1669,6 +1677,25 @@ export async function crawlProject(opts: {
             await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
           }
 
+          // Wait for SPA JS to render components before extracting elements or running soft-404 check.
+          // domcontentloaded fires before JS executes — networkidle ensures React/Vue/Angular hydration.
+          await workerPage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+
+          // Soft-404 detection for lite-worker: skip SPA catch-all URLs that mirror homepage
+          if (homeFingerprintElemCount >= 0 && url !== startUrl) {
+            const wTitle = await workerPage.title().catch(() => '');
+            if (wTitle === homeFingerprintTitle && wTitle !== '') {
+              const wElemCount = await workerPage.evaluate(() =>
+                document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+              ).catch(() => -1);
+              if (wElemCount >= 0 && wElemCount === homeFingerprintElemCount) {
+                console.log(`[lite-worker] skip ${url} — soft-404 (homepage fingerprint match)`);
+                visited.set(canonUrl, '_soft404_');
+                continue;
+              }
+            }
+          }
+
           const pathParts = new URL(url).pathname.split('/').filter(Boolean);
           const name = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
 
@@ -2157,6 +2184,18 @@ export async function crawlProject(opts: {
           seenDomHashes.add(preDomHash); // register so future 404 pages also hit hash-dedup
           continue;
         }
+        // Soft-404: SPA catch-all routes return 200 but render identical homepage content.
+        // Skip any non-homepage URL whose title + interactive element count matches the homepage.
+        if (homeFingerprintElemCount >= 0 && url !== startUrl && pageTitle === homeFingerprintTitle && pageTitle !== '') {
+          const pElemCount = await page.evaluate(() =>
+            document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+          ).catch(() => -1);
+          if (pElemCount >= 0 && pElemCount === homeFingerprintElemCount) {
+            console.log(`[crawler] skip ${url} — soft-404 (homepage fingerprint match, title="${pageTitle}" elems=${pElemCount})`);
+            seenDomHashes.add(preDomHash);
+            continue;
+          }
+        }
         // Do NOT add real pages to seenDomHashes — server-rendered sites (WooCommerce, WordPress)
         // use the same template for many legitimate pages; registering the hash would skip them all.
 
@@ -2537,11 +2576,14 @@ export async function crawlProject(opts: {
                   const snippet = extracted.markdown.slice(0, 1200);
                   // Skip LLM when content is too thin — output would be useless
                   if (snippet.trim().length < 80) return undefined;
+                  const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ''}:page-summary:${url}`);
                   const { text, usage } = await askTracked(
                     'You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.',
                     `Screen: "${name}"\nURL: ${url}\n\nContent:\n${snippet}`,
                     llmCfg,
                     200,
+                    undefined,
+                    { temperature: 0, seed: summarySeed },
                   );
                   if (usage) {
                     recordLLMCall({
@@ -4045,13 +4087,23 @@ async function domFallbackElements(page: any): Promise<Array<{ meaning: string;
  *  Headings, meta tags, cards, chips, FAQ items, nav/footer structure, section text.
  *  Previously fallback-only; now always runs alongside domFallbackElements so
  *  the test generator sees ALL page content, not just interactive widgets. */
-async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string }>> {
+async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number> }>> {
   return page.evaluate(() => {
-    const out: { meaning: string; role: string; notes?: string }[] = [];
+    const out: { meaning: string; role: string; notes?: string; boundingRect?: Record<string, number> }[] = [];
     const seen = new Set<string>();
-    const addUniq = (item: { meaning: string; role: string; notes?: string }) => {
+    const captureRect = (el: Element): Record<string, number> | undefined => {
+      try {
+        const r = el.getBoundingClientRect();
+        if (r.width === 0 && r.height === 0) return undefined;
+        return { x: r.x, y: r.y, width: r.width, height: r.height, top: r.top, right: r.right, bottom: r.bottom, left: r.left };
+      } catch { return undefined; }
+    };
+    const addUniq = (item: { meaning: string; role: string; notes?: string }, el?: Element) => {
       const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
-      if (!seen.has(key) && item.meaning.trim().length > 1) { seen.add(key); out.push(item); }
+      if (!seen.has(key) && item.meaning.trim().length > 1) {
+        seen.add(key);
+        out.push(el ? { ...item, boundingRect: captureRect(el) } : item);
+      }
     };
     const isVisible = (el: Element) => {
       const s = window.getComputedStyle(el);
@@ -4072,7 +4124,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((h: any) => {
       if (!isVisible(h)) return;
       const text = h.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
-      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) });
+      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) }, h);
     });
 
     // 3. FAQ / accordion — <details>/<summary>, ARIA pattern, and class-name scan
@@ -4113,7 +4165,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('[class*="card" i]:not(nav):not(header):not(footer), [class*="chip" i], [class*="badge" i], [class*="tag" i], [class*="pill" i]').forEach((el: any) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' });
+      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' }, el);
     });
 
     // 5. Feature / service list items inside main content (capped per category)
@@ -4123,14 +4175,14 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
       if (listItemCount >= 30) return;
       if (!isVisible(li)) return;
       const text = li.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }); listItemCount++; }
+      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }, li); listItemCount++; }
     });
 
     // 6. First paragraph in each section (key descriptive copy)
     mainContent.querySelectorAll('section > p:first-of-type, article > p:first-of-type, [class*="section" i] > p:first-of-type').forEach((p: any) => {
       if (!isVisible(p)) return;
       const text = p.textContent?.replace(/\s+/g, ' ').trim().slice(0, 250);
-      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' });
+      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' }, p);
     });
 
     // 7. CTA section text (standalone paragraphs near buttons that aren't form fields)
@@ -4562,7 +4614,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('[role="alert"], [role="status"], [role="banner"]:not(header), .alert, .notice, .notification, .warning-banner, .error-banner, .success-banner, [class*="alert-" i]:not(button), [class*="toast" i]:not(button), [class*="snackbar" i]').forEach((el: any) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) });
+      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) }, el);
     });
 
     // 27. Pricing / plan cards — structured pricing content
@@ -5339,6 +5391,21 @@ async function runAuthDiscoveryPhase(
     try { enqueue(new URL(p, origin).toString(), 'seed_path'); } catch { /* ignore bad seed */ }
   }
 
+  // Capture homepage DOM fingerprint for soft-404 detection.
+  // SPA catch-all sites return 200 for every path but render identical content.
+  // If a seeded path produces the same element count + title as the homepage, it's
+  // a phantom route — skip it so auth paths don't pollute the discovered URL list.
+  let homeTitle = '';
+  let homeElemCount = -1;
+  try {
+    await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => null);
+    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
+    homeTitle = await page.title().catch(() => '');
+    homeElemCount = await page.evaluate(() =>
+      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
+  } catch { /* best-effort */ }
+
   let captured = 0;
   let ordinal = 0;
   await onProgress?.(10, 'Discovering public and auth flows');
@@ -5352,6 +5419,16 @@ async function runAuthDiscoveryPhase(
       await waitForStableDOM(page);
       await acceptCookieConsent(page, { waitMs: 400 }).catch(() => { });
 
+      // Soft-404 detection: SPA returns 200 for unknown paths but renders homepage.
+      // Skip if title + element count match the homepage fingerprint exactly.
+      if (current.source === 'seed_path' && homeElemCount >= 0) {
+        const pageTitle = await page.title().catch(() => '');
+        const elemCount = await page.evaluate(() =>
+          document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+        ).catch(() => -1);
+        if (pageTitle === homeTitle && elemCount === homeElemCount) continue;
+      }
+
       const finalUrl = page.url?.() ?? current.url;
       const captureKey = normalizeAuthUrl(finalUrl);
       if (!capturedKeys.has(captureKey)) {

package/src/failure-analysis-agent.ts

@@ -1,5 +1,5 @@
 import { readFileSync } from 'node:fs';
-import { askTrackedWithImages, askTracked } from '@detiq/agents';
+import { askTrackedWithImages, askTracked, deterministicSeed } from '@detiq/agents';
 import type { LLMConfig } from '@detiq/agents';
 import { AppBrain } from '@detiq/app-brain';
 
@@ -41,7 +41,8 @@ Error: ${ctx.error}${consoleSection}${networkSection}`;
       } catch { /* fall through to text-only */ }
     }
 
-    const result = await askTracked('', userText, llmCfg, 256);
+    const failureSeed = deterministicSeed(`${ctx.tenantId ?? ''}:failure:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+    const result = await askTracked('', userText, llmCfg, 256, undefined, { temperature: 0, seed: failureSeed });
     return result.text || null;
   } catch {
     return null;

package/src/gap-pass.ts

@@ -75,6 +75,7 @@ Respond with ONLY valid JSON, no markdown:
     const response = await client.messages.create({
       model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore
       system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
       messages: [{ role: 'user', content: prompt }],

package/src/job-queue.ts

@@ -1273,7 +1273,7 @@ async function processGenerateTestsFromReqsJob(jobId: string, config: any) {
   try {
     await updateJobProgress(tenantId, jobId, 5, 'Loading requirements');
 
-    const { askTracked, getTaskAgentLLMConfig } = await import('@detiq/agents');
+    const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import('@detiq/agents');
     const llmCfg = await getTaskAgentLLMConfig(tenantId, 'testGeneration');
 
     const all = await AppBrain.listRequirements(tenantId, projectId, {}) as any[];
@@ -1321,7 +1321,8 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
 
     let parsed: { cases: any[] } = { cases: [] };
     try {
-      const { text, usage } = await askTracked(system, user, llmCfg);
+      const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
+      const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
       if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => {});
       const json = text.match(/\{[\s\S]*\}/)?.[0];
       if (json) parsed = JSON.parse(json);

package/src/url-pattern-completer.ts

@@ -72,6 +72,7 @@ Respond with ONLY valid JSON, no explanation:
     const response = await client.messages.create({
       model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore: cache_control supported in beta
       system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
       messages: [{ role: 'user', content: prompt }],