@detiq/crawler 1.1.2 → 1.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.2",
+  "version": "1.1.6",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -28,7 +28,7 @@ import { z } from 'zod';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { AppBrain, recordLLMCall } from '@detiq/app-brain';
-import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
+import { aiVisualCompare, askTracked, deterministicSeed, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
 import { config } from '@detiq/core';
 import { pHashVerdict, mergeAiVisualVerdict } from './visual-diff-verdict.js';
 import type { Credentials } from './auth.js';
@@ -1125,6 +1125,14 @@ export async function crawlProject(opts: {
     await page.waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
     await waitForStableDOM(page);
 
+    // Capture homepage fingerprint for soft-404 detection. SPA catch-all routes return HTTP 200
+    // but render identical homepage content for every unrecognised path. We fingerprint here
+    // (pre-auth, stable DOM) and skip any BFS page that matches during the main crawl.
+    const homeFingerprintTitle = await page.title().catch(() => '');
+    const homeFingerprintElemCount = await page.evaluate(() =>
+      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
+
     // Dedicated logged-out/auth discovery phase. This runs before credentials or
     // storageState are applied, so public pages and auth recovery flows remain
     // visible even when the main crawl later becomes authenticated.
@@ -1638,6 +1646,111 @@ export async function crawlProject(opts: {
 
     let pendingSessionRestore: (() => Promise<void>) | null = null;
 
+    // F10: Parallel lite-worker browser tabs — pull from shared queue while main loop runs.
+    // Each worker uses a fresh Playwright page from the same browser context so it shares
+    // cookies/session with the main page. DOM-only extraction (no Stagehand AI).
+    // Default 5 = 3-5× crawl throughput on a single browser context. Set CRAWL_PAGE_PARALLELISM=1 to disable.
+    const CRAWL_PAGE_PARALLELISM = Math.max(1, parseInt(process.env.CRAWL_PAGE_PARALLELISM ?? '5', 10));
+
+    async function runLiteWorker(workerPage: any): Promise<void> {
+      while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+        if (opts.abortSignal?.aborted) break;
+        if (++totalAttemptsThisSegment > maxAttempts) break;
+
+        const url = queue.shift();
+        if (!url) break;
+
+        const canonUrl = canonicalize(url);
+        if (visited.has(canonUrl)) continue;
+
+        try {
+          const { pathname } = new URL(url);
+          if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
+        } catch { continue; }
+
+        // Claim URL before first await — prevents duplicate processing across parallel workers
+        visited.set(canonUrl, '_pending_');
+
+        try {
+          const navErr = await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null as null, (e: any) => e);
+          if (navErr && /timeout/i.test(String((navErr as any)?.message ?? navErr))) {
+            await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
+          }
+
+          // Wait for SPA JS to render components before extracting elements or running soft-404 check.
+          // domcontentloaded fires before JS executes — networkidle ensures React/Vue/Angular hydration.
+          await workerPage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+
+          // Soft-404 detection for lite-worker: skip SPA catch-all URLs that mirror homepage
+          if (homeFingerprintElemCount >= 0 && url !== startUrl) {
+            const wTitle = await workerPage.title().catch(() => '');
+            if (wTitle === homeFingerprintTitle && wTitle !== '') {
+              const wElemCount = await workerPage.evaluate(() =>
+                document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+              ).catch(() => -1);
+              if (wElemCount >= 0 && wElemCount === homeFingerprintElemCount) {
+                console.log(`[lite-worker] skip ${url} — soft-404 (homepage fingerprint match)`);
+                visited.set(canonUrl, '_soft404_');
+                continue;
+              }
+            }
+          }
+
+          const pathParts = new URL(url).pathname.split('/').filter(Boolean);
+          const name = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
+
+          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
+          visited.set(canonUrl, screen.id);
+
+          const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
+
+          const [elements] = await Promise.all([
+            domFallbackElements(workerPage).catch(() => [] as Awaited<ReturnType<typeof domFallbackElements>>),
+            workerPage.screenshot({ path: shotPath, fullPage: true })
+              .then(async () => {
+                const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+                return (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl);
+              })
+              .catch(() => {}),
+          ]);
+
+          const nextLinks: string[] = await (workerPage.evaluate as any)((originStr: string) => {
+            return Array.from(document.querySelectorAll('a[href]'))
+              .map((a: any) => {
+                try { return new URL(a.getAttribute('href')!, window.location.href).href; }
+                catch { return ''; }
+              })
+              .filter((h: string) => h && new URL(h).origin === originStr && !h.includes('#'))
+              .slice(0, 200);
+          }, origin).catch(() => []);
+
+          for (const link of nextLinks) {
+            const cLink = canonicalize(link);
+            if (!visited.has(cLink)) {
+              queue.push(link);
+              edges.push({ fromUrl: url, toUrl: link });
+            }
+          }
+
+          discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
+        } catch (err: any) {
+          logger.warn({ url, err: err?.message ?? String(err) }, '[lite-worker] URL processing failed');
+          discovered.push({ url, error: err?.message ?? String(err) });
+        }
+      }
+    }
+
+    const liteWorkerPromises: Promise<void>[] = [];
+    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser) {
+      const pwCtx = playwrightBrowser.contexts()[0];
+      for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
+        const wp = await pwCtx.newPage();
+        await wp.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }).catch(() => {});
+        wp.setDefaultTimeout(45_000);
+        liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {})));
+      }
+    }
+
     while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
       if (pendingSessionRestore) {
         await pendingSessionRestore();
@@ -1767,9 +1880,17 @@ export async function crawlProject(opts: {
             }
           }
 
-          await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
-            logger.warn({ url, err: navErr?.message ?? String(navErr) }, '[crawler] navigation failed — skipping');
-          });
+          // Try fast 15 s first; only escalate to 30 s on timeout so happy-path pages don't wait
+          const navErr15 = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null, (e: any) => e);
+          if (navErr15) {
+            if (/timeout/i.test(String(navErr15?.message ?? navErr15))) {
+              await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
+                logger.warn({ url, err: navErr?.message ?? String(navErr) }, '[crawler] navigation failed — skipping');
+              });
+            } else {
+              logger.warn({ url, err: navErr15?.message ?? String(navErr15) }, '[crawler] navigation failed — skipping');
+            }
+          }
           rawPage.off('response', onRedirectResponse);
           if (redirectHops.length > 0) {
             redirectRegistry.register({
@@ -2063,6 +2184,18 @@ export async function crawlProject(opts: {
           seenDomHashes.add(preDomHash); // register so future 404 pages also hit hash-dedup
           continue;
         }
+        // Soft-404: SPA catch-all routes return 200 but render identical homepage content.
+        // Skip any non-homepage URL whose title + interactive element count matches the homepage.
+        if (homeFingerprintElemCount >= 0 && url !== startUrl && pageTitle === homeFingerprintTitle && pageTitle !== '') {
+          const pElemCount = await page.evaluate(() =>
+            document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+          ).catch(() => -1);
+          if (pElemCount >= 0 && pElemCount === homeFingerprintElemCount) {
+            console.log(`[crawler] skip ${url} — soft-404 (homepage fingerprint match, title="${pageTitle}" elems=${pElemCount})`);
+            seenDomHashes.add(preDomHash);
+            continue;
+          }
+        }
         // Do NOT add real pages to seenDomHashes — server-rendered sites (WooCommerce, WordPress)
         // use the same template for many legitimate pages; registering the hash would skip them all.
 
@@ -2350,7 +2483,8 @@ export async function crawlProject(opts: {
           }),
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
-          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0)
+          // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
+          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < 5)
             .then((freshLinks) => {
               if (aiSkipCount > 0) aiSkipCount--;
               aiErrorStreak = 0;
@@ -2442,11 +2576,14 @@ export async function crawlProject(opts: {
                   const snippet = extracted.markdown.slice(0, 1200);
                   // Skip LLM when content is too thin — output would be useless
                   if (snippet.trim().length < 80) return undefined;
+                  const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ''}:page-summary:${url}`);
                   const { text, usage } = await askTracked(
                     'You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.',
                     `Screen: "${name}"\nURL: ${url}\n\nContent:\n${snippet}`,
                     llmCfg,
                     200,
+                    undefined,
+                    { temperature: 0, seed: summarySeed },
                   );
                   if (usage) {
                     recordLLMCall({
@@ -2909,6 +3046,9 @@ export async function crawlProject(opts: {
       }
     }
 
+    // Wait for any lite workers still processing URLs from the shared queue
+    if (liteWorkerPromises.length > 0) await Promise.all(liteWorkerPromises);
+
     // Discovery mode: return collected URLs and skip all post-crawl analysis phases.
     // Use visited.values() (original URLs) not .keys() (canonical forms with /{id} placeholders).
     if (opts.discoveryMode) {
@@ -3947,13 +4087,23 @@ async function domFallbackElements(page: any): Promise<Array<{ meaning: string;
  *  Headings, meta tags, cards, chips, FAQ items, nav/footer structure, section text.
  *  Previously fallback-only; now always runs alongside domFallbackElements so
  *  the test generator sees ALL page content, not just interactive widgets. */
-async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string }>> {
+async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number> }>> {
   return page.evaluate(() => {
-    const out: { meaning: string; role: string; notes?: string }[] = [];
+    const out: { meaning: string; role: string; notes?: string; boundingRect?: Record<string, number> }[] = [];
     const seen = new Set<string>();
-    const addUniq = (item: { meaning: string; role: string; notes?: string }) => {
+    const captureRect = (el: Element): Record<string, number> | undefined => {
+      try {
+        const r = el.getBoundingClientRect();
+        if (r.width === 0 && r.height === 0) return undefined;
+        return { x: r.x, y: r.y, width: r.width, height: r.height, top: r.top, right: r.right, bottom: r.bottom, left: r.left };
+      } catch { return undefined; }
+    };
+    const addUniq = (item: { meaning: string; role: string; notes?: string }, el?: Element) => {
       const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
-      if (!seen.has(key) && item.meaning.trim().length > 1) { seen.add(key); out.push(item); }
+      if (!seen.has(key) && item.meaning.trim().length > 1) {
+        seen.add(key);
+        out.push(el ? { ...item, boundingRect: captureRect(el) } : item);
+      }
     };
     const isVisible = (el: Element) => {
       const s = window.getComputedStyle(el);
@@ -3974,7 +4124,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((h: any) => {
       if (!isVisible(h)) return;
       const text = h.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
-      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) });
+      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) }, h);
     });
 
     // 3. FAQ / accordion — <details>/<summary>, ARIA pattern, and class-name scan
@@ -4015,7 +4165,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('[class*="card" i]:not(nav):not(header):not(footer), [class*="chip" i], [class*="badge" i], [class*="tag" i], [class*="pill" i]').forEach((el: any) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' });
+      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' }, el);
     });
 
     // 5. Feature / service list items inside main content (capped per category)
@@ -4025,14 +4175,14 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
       if (listItemCount >= 30) return;
       if (!isVisible(li)) return;
       const text = li.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }); listItemCount++; }
+      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }, li); listItemCount++; }
     });
 
     // 6. First paragraph in each section (key descriptive copy)
     mainContent.querySelectorAll('section > p:first-of-type, article > p:first-of-type, [class*="section" i] > p:first-of-type').forEach((p: any) => {
       if (!isVisible(p)) return;
       const text = p.textContent?.replace(/\s+/g, ' ').trim().slice(0, 250);
-      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' });
+      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' }, p);
     });
 
     // 7. CTA section text (standalone paragraphs near buttons that aren't form fields)
@@ -4464,7 +4614,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('[role="alert"], [role="status"], [role="banner"]:not(header), .alert, .notice, .notification, .warning-banner, .error-banner, .success-banner, [class*="alert-" i]:not(button), [class*="toast" i]:not(button), [class*="snackbar" i]').forEach((el: any) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) });
+      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) }, el);
     });
 
     // 27. Pricing / plan cards — structured pricing content
@@ -5052,7 +5202,7 @@ async function hasMeaningfulAuthOrPublicState(page: any, requestedUrl: string):
     }
   })();
 
-  const domLooksRelevant = await page.evaluate(() => {
+  const domState = await page.evaluate(() => {
     const text = (document.body?.innerText ?? '').toLowerCase();
     const inputs = Array.from(document.querySelectorAll('input')) as HTMLInputElement[];
     const hasPassword = inputs.some((i) => i.type === 'password');
@@ -5062,9 +5212,16 @@ async function hasMeaningfulAuthOrPublicState(page: any, requestedUrl: string):
     });
     const hasAuthCopy = /(sign in|log in|login|sign up|create account|register|forgot password|reset password|verify email|one-time|otp|continue with email)/i.test(text);
     const hasPublicNav = /(pricing|contact|features|documentation|docs|about|privacy|terms)/i.test(text);
-    return hasPassword || hasIdentityInput || hasAuthCopy || hasPublicNav;
-  }).catch(() => false);
+    // Soft-404: page says "not found" / "404" with no real auth form
+    const isSoft404 = !hasPassword && !hasIdentityInput &&
+      /(page not found|404|this page (could not|doesn'?t) (be found|exist)|we couldn'?t find|no longer exists|been moved|doesn'?t exist)/i.test(text);
+    return { hasPassword, hasIdentityInput, hasAuthCopy, hasPublicNav, isSoft404 };
+  }).catch(() => ({ hasPassword: false, hasIdentityInput: false, hasAuthCopy: false, hasPublicNav: false, isSoft404: false }));
+
+  // Soft-404 with no auth form — path match alone is not enough
+  if (domState.isSoft404) return false;
 
+  const domLooksRelevant = domState.hasPassword || domState.hasIdentityInput || domState.hasAuthCopy || domState.hasPublicNav;
   return pathLooksRelevant || domLooksRelevant;
 }
 
@@ -5234,6 +5391,21 @@ async function runAuthDiscoveryPhase(
     try { enqueue(new URL(p, origin).toString(), 'seed_path'); } catch { /* ignore bad seed */ }
   }
 
+  // Capture homepage DOM fingerprint for soft-404 detection.
+  // SPA catch-all sites return 200 for every path but render identical content.
+  // If a seeded path produces the same element count + title as the homepage, it's
+  // a phantom route — skip it so auth paths don't pollute the discovered URL list.
+  let homeTitle = '';
+  let homeElemCount = -1;
+  try {
+    await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => null);
+    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
+    homeTitle = await page.title().catch(() => '');
+    homeElemCount = await page.evaluate(() =>
+      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
+  } catch { /* best-effort */ }
+
   let captured = 0;
   let ordinal = 0;
   await onProgress?.(10, 'Discovering public and auth flows');
@@ -5247,6 +5419,16 @@ async function runAuthDiscoveryPhase(
       await waitForStableDOM(page);
       await acceptCookieConsent(page, { waitMs: 400 }).catch(() => { });
 
+      // Soft-404 detection: SPA returns 200 for unknown paths but renders homepage.
+      // Skip if title + element count match the homepage fingerprint exactly.
+      if (current.source === 'seed_path' && homeElemCount >= 0) {
+        const pageTitle = await page.title().catch(() => '');
+        const elemCount = await page.evaluate(() =>
+          document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+        ).catch(() => -1);
+        if (pageTitle === homeTitle && elemCount === homeElemCount) continue;
+      }
+
       const finalUrl = page.url?.() ?? current.url;
       const captureKey = normalizeAuthUrl(finalUrl);
       if (!capturedKeys.has(captureKey)) {

package/src/failure-analysis-agent.ts

@@ -1,5 +1,5 @@
 import { readFileSync } from 'node:fs';
-import { askTrackedWithImages, askTracked } from '@detiq/agents';
+import { askTrackedWithImages, askTracked, deterministicSeed } from '@detiq/agents';
 import type { LLMConfig } from '@detiq/agents';
 import { AppBrain } from '@detiq/app-brain';
 
@@ -41,7 +41,8 @@ Error: ${ctx.error}${consoleSection}${networkSection}`;
       } catch { /* fall through to text-only */ }
     }
 
-    const result = await askTracked('', userText, llmCfg, 256);
+    const failureSeed = deterministicSeed(`${ctx.tenantId ?? ''}:failure:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+    const result = await askTracked('', userText, llmCfg, 256, undefined, { temperature: 0, seed: failureSeed });
     return result.text || null;
   } catch {
     return null;

package/src/gap-pass.ts

@@ -75,6 +75,7 @@ Respond with ONLY valid JSON, no markdown:
     const response = await client.messages.create({
       model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore
       system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
       messages: [{ role: 'user', content: prompt }],

package/src/job-queue.ts

@@ -191,7 +191,7 @@ const runningControllers = new Map<string, AbortController>();
 const WORKER_ID = `${os.hostname()}:${process.pid}`;
 const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
 const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15_000;
-const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 3;
+const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
 const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15_000;
 // How many headless browsers a single RUN_TEST_SUITE job launches at once —
 // distinct from WORKER_CONCURRENCY (how many *jobs* the poll loop claims at
@@ -199,7 +199,7 @@ const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS)
 // primary scale-out path for running more tests at once is still horizontal
 // (more worker containers / SQS), not raising this past what one container
 // can safely hold — see startJobWorkerLoop's doc comment.
-const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 3;
+const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 5;
 
 type ClaimedJob = { id: string; tenantId: string; projectId: string; type: string; config: any };
 
@@ -1273,7 +1273,7 @@ async function processGenerateTestsFromReqsJob(jobId: string, config: any) {
   try {
     await updateJobProgress(tenantId, jobId, 5, 'Loading requirements');
 
-    const { askTracked, getTaskAgentLLMConfig } = await import('@detiq/agents');
+    const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import('@detiq/agents');
     const llmCfg = await getTaskAgentLLMConfig(tenantId, 'testGeneration');
 
     const all = await AppBrain.listRequirements(tenantId, projectId, {}) as any[];
@@ -1321,7 +1321,8 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
 
     let parsed: { cases: any[] } = { cases: [] };
     try {
-      const { text, usage } = await askTracked(system, user, llmCfg);
+      const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
+      const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
       if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => {});
       const json = text.match(/\{[\s\S]*\}/)?.[0];
       if (json) parsed = JSON.parse(json);

package/src/sitemap.ts

@@ -111,9 +111,11 @@ export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth
   const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
   const collected = [...pageUrls];
 
-  for (const subUrl of subSitemapUrls) {
-    if (collected.length >= MAX_URLS) break;
-    collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));
+  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+    );
+    for (const sub of subResults) collected.push(...sub);
   }
 
   return [...new Set(collected)].slice(0, MAX_URLS);

package/src/url-pattern-completer.ts

@@ -72,6 +72,7 @@ Respond with ONLY valid JSON, no explanation:
     const response = await client.messages.create({
       model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore: cache_control supported in beta
       system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
       messages: [{ role: 'user', content: prompt }],