@detiq/crawler 1.1.21 → 1.1.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.21",
+  "version": "1.1.23",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -102,7 +102,7 @@ const _crawlLLMCache = new Map<string, any>();
 async function getCachedCrawlLLM(tenantId: string): Promise<any> {
   if (_crawlLLMCache.has(tenantId)) return _crawlLLMCache.get(tenantId);
   const cfg = await getTaskAgentLLMConfig(tenantId, 'crawl').catch(() => null);
-  _crawlLLMCache.set(tenantId, cfg);
+  if (cfg) _crawlLLMCache.set(tenantId, cfg);
   return cfg;
 }
 
@@ -128,7 +128,9 @@ interface PerformanceMetric {
 const ALLOWED_NUMERIC_PATHS = new Set(['/2fa', '/404', '/500', '/403', '/401']);
 const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
-const NUMERIC_ID_PATH_SEGMENT_RE = /^\d[\d-]*$/;
+// Match 4+ digit pure numeric IDs only — aligns with canonicalize()'s /\d{4,}/ threshold.
+// 1-3 digit segments (page/2/, /tag/books/page/3/) are valid pagination — do NOT skip them.
+const NUMERIC_ID_PATH_SEGMENT_RE = /^\d{4,}$/;
 const COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE = /^\d+-\d+$/;
 
 function normalizePathForNumericGuard(pathname: string): string {
@@ -1955,6 +1957,7 @@ export async function crawlProject(opts: {
     };
     const edges: Array<{ fromUrl: string; toUrl: string }> = [];
     const pendingTransitions: Array<{ fromScreenId: string; elementMeaning: string; targetUrl: string }> = [];
+    const pendingSummaryWrites: Promise<void>[] = [];
     const discovered: Array<{
       url: string; screenId?: string; elements?: number;
       elementFacts?: Array<{ meaning: string; role: string; expectedData?: string }>;
@@ -3450,6 +3453,7 @@ export async function crawlProject(opts: {
               const summaryPromise = (async () => {
                 try {
                   const llmCfg = await getCachedCrawlLLM(opts.tenantId);
+                  if (!llmCfg) return undefined;
                   const snippet = extracted.markdown.slice(0, 1200);
                   // Skip LLM when content is too thin — output would be useless
                   if (snippet.trim().length < 80) return undefined;
@@ -3487,9 +3491,15 @@ export async function crawlProject(opts: {
                 heading: extracted.title || name,
               }).catch((err: any) => console.warn('[crawler] markdown extraction save failed:', err));
 
-              summaryPromise.then((aiSummary) => {
-                if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch(() => { });
-              });
+              pendingSummaryWrites.push(
+                summaryPromise.then((aiSummary) => {
+                  if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch((e: any) => {
+                    logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, '[crawler] aiSummary DB save failed');
+                  });
+                }).catch((e: any) => {
+                  logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, '[crawler] aiSummary generation failed');
+                })
+              );
             }
 
             const structuredJson = summarizeStructuredElements(atElements);
@@ -3989,9 +3999,11 @@ export async function crawlProject(opts: {
 
     // Discovery mode: return collected URLs and skip all post-crawl analysis phases.
     // Use visited.values() (original URLs) not .keys() (canonical forms with /{id} placeholders).
+    // Filter out sentinel strings (_pending_, _auth_wall_, _soft404_, etc.) that mark
+    // in-progress or skipped entries — only keep actual http(s) URLs.
     if (opts.discoveryMode) {
       return {
-        discoveredUrls: Array.from(visited.values()),
+        discoveredUrls: Array.from(visited.values()).filter((v: string) => v.startsWith('http')),
         screensFound: 0,
         screensErrored: 0,
         screensSkippedToNextSegment: 0,
@@ -4768,6 +4780,15 @@ export async function crawlProject(opts: {
     const screensSucceeded = discovered.filter(d => !!d.screenId).length;
     const screensErrored = discovered.filter(d => !!d.error && !d.screenId).length;
 
+    // Drain pending AI summary writes before marking the job done — ensures descriptions
+    // are visible immediately after crawl completes. Cap wait at 30 s per screen, 60 s total.
+    if (pendingSummaryWrites.length > 0) {
+      await Promise.race([
+        Promise.allSettled(pendingSummaryWrites),
+        new Promise<void>((resolve) => setTimeout(resolve, Math.min(60_000, pendingSummaryWrites.length * 3_000))),
+      ]);
+    }
+
     // Detect systematic element/link extraction failure across all screens
     if (screensSucceeded >= 3) {
       const screensWithElements = discovered.filter(d => (d.elements ?? 0) > 0 && d.screenId).length;

package/src/job-queue.ts

@@ -86,7 +86,7 @@ export function generateTOTP(secret: string): string {
   const hmac = createHmac('sha1', Buffer.from(bytes)).update(buf).digest();
   const offset = hmac[hmac.length - 1] & 0xf;
   const code = ((hmac[offset] & 0x7f) << 24) | ((hmac[offset + 1] & 0xff) << 16) |
-               ((hmac[offset + 2] & 0xff) << 8) | (hmac[offset + 3] & 0xff);
+    ((hmac[offset + 2] & 0xff) << 8) | (hmac[offset + 3] & 0xff);
   return String(code % 1_000_000).padStart(6, '0');
 }
 
@@ -94,12 +94,12 @@ function deriveFallbackTargetPath(ref: string, framework: string): string {
   const slug = (ref ?? 'test').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'test';
   const base = framework.startsWith('cypress') ? 'cypress/e2e/'
     : (framework === 'playwright-py' || framework === 'pytest' || framework === 'selenium-py') ? 'tests/'
-    : (framework === 'playwright-java' || framework === 'selenium-java' || framework === 'junit') ? 'src/test/java/'
-    : 'tests/';
+      : (framework === 'playwright-java' || framework === 'selenium-java' || framework === 'junit') ? 'src/test/java/'
+        : 'tests/';
   const ext = (framework === 'playwright-py' || framework === 'pytest' || framework === 'selenium-py') ? '_test.py'
     : (framework === 'playwright-java' || framework === 'selenium-java' || framework === 'junit') ? 'Test.java'
-    : framework.includes('ts') ? '.spec.ts'
-    : '.spec.js';
+      : framework.includes('ts') ? '.spec.ts'
+        : '.spec.js';
   return `${base}${slug}${ext}`;
 }
 
@@ -109,23 +109,23 @@ function makeJobName(type: string): string {
   const ts = now.getTime();
   const label = type === 'CRAWL' ? 'Crawl'
     : type === 'CAPTURE_STEPS' ? 'CaptureSteps'
-    : type === 'NATIVE_CRAWL' ? 'NativeCrawl'
-    : type === 'GENERATE_TESTS' ? 'GenerateTests'
-    : type === 'VISUAL_REGRESSION' ? 'VisualRegression'
-    : type === 'AI_EDIT' ? 'AIEdit'
-    : type === 'API_TESTS' ? 'ApiTests'
-    : type === 'SECURITY_TESTS' ? 'SecurityTests'
-    : type === 'INTEGRATION_TESTS' ? 'IntegrationTests'
-    : type === 'GENERATE_TESTS_FROM_REQS' ? 'GenerateTestsFromReqs'
-    : type === 'GENERATE_DATA' ? 'GenerateData'
-    : type === 'GENERATE_SCRIPT' ? 'GenerateScript'
-    : type === 'GENERATE_TEST_PLAN' ? 'GenerateTestPlan'
-    : type === 'IMPORT_TESTS' ? 'ImportTests'
-    : type === 'GENERATE_SCREEN_TESTS' ? 'GenerateScreenTests'
-    : type === 'BATCH_GENERATE_ALIGNED' ? 'BatchGenerateAligned'
-    : type === 'CREATE_PR' ? 'CreatePR'
-    : type === 'NEW_PAGES_PIPELINE' ? 'NewPagesPipeline'
-    : type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+      : type === 'NATIVE_CRAWL' ? 'NativeCrawl'
+        : type === 'GENERATE_TESTS' ? 'GenerateTests'
+          : type === 'VISUAL_REGRESSION' ? 'VisualRegression'
+            : type === 'AI_EDIT' ? 'AIEdit'
+              : type === 'API_TESTS' ? 'ApiTests'
+                : type === 'SECURITY_TESTS' ? 'SecurityTests'
+                  : type === 'INTEGRATION_TESTS' ? 'IntegrationTests'
+                    : type === 'GENERATE_TESTS_FROM_REQS' ? 'GenerateTestsFromReqs'
+                      : type === 'GENERATE_DATA' ? 'GenerateData'
+                        : type === 'GENERATE_SCRIPT' ? 'GenerateScript'
+                          : type === 'GENERATE_TEST_PLAN' ? 'GenerateTestPlan'
+                            : type === 'IMPORT_TESTS' ? 'ImportTests'
+                              : type === 'GENERATE_SCREEN_TESTS' ? 'GenerateScreenTests'
+                                : type === 'BATCH_GENERATE_ALIGNED' ? 'BatchGenerateAligned'
+                                  : type === 'CREATE_PR' ? 'CreatePR'
+                                    : type === 'NEW_PAGES_PIPELINE' ? 'NewPagesPipeline'
+                                      : type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
   return `${label}_${date}_${ts}`;
 }
 
@@ -195,6 +195,9 @@ const WORKER_ID = `${os.hostname()}:${process.pid}`;
 const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
 const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15_000;
 const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
+// Max simultaneous DISCOVER jobs — each launches a Playwright browser and holds
+// DB connections, so running many concurrently exhausts resources. Default 2.
+const MAX_CONCURRENT_DISCOVER = Number(process.env.MAX_CONCURRENT_DISCOVER) || 2;
 const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15_000;
 // How many headless browsers a single RUN_TEST_SUITE job launches at once —
 // distinct from WORKER_CONCURRENCY (how many *jobs* the poll loop claims at
@@ -221,14 +224,21 @@ export async function claimSpecificJob(jobId: string): Promise<ClaimedJob | null
 }
 
 /** Atomically claims the oldest QUEUED job across all tenants. SKIP LOCKED makes this
- *  race-safe against any number of other pollers doing the same thing concurrently. */
+ *  race-safe against any number of other pollers doing the same thing concurrently.
+ *  DISCOVER jobs are subject to MAX_CONCURRENT_DISCOVER — skipped when that many are
+ *  already RUNNING to prevent Playwright/DB pool exhaustion from simultaneous crawls. */
 export async function claimNextQueuedJob(): Promise<ClaimedJob | null> {
+  const maxDiscover = MAX_CONCURRENT_DISCOVER;
   const rows = await prismaAdmin.$queryRaw<ClaimedJob[]>`
     UPDATE "PlatformJob"
     SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
     WHERE id = (
       SELECT id FROM "PlatformJob"
       WHERE status = 'QUEUED'
+        AND (
+          type != 'DISCOVER'
+          OR (SELECT COUNT(*) FROM "PlatformJob" WHERE status = 'RUNNING' AND type = 'DISCOVER') < ${maxDiscover}
+        )
       ORDER BY "queuedAt" ASC
       FOR UPDATE SKIP LOCKED
       LIMIT 1
@@ -315,7 +325,7 @@ async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
     await prismaAdmin.platformJob.update({
       where: { id: job.id },
       data: { status: 'FAILED', error: `Unknown job type: ${job.type}`, finishedAt: new Date() },
-    }).catch(() => {});
+    }).catch(() => { });
     return;
   }
   const stopTimer = crawlJobDurationSeconds.startTimer({ type: job.type });
@@ -335,7 +345,7 @@ async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
     await prismaAdmin.platformJob.update({
       where: { id: job.id },
       data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() },
-    }).catch(() => {});
+    }).catch(() => { });
   }
 }
 
@@ -362,10 +372,15 @@ export function startJobWorkerLoop(opts?: { pollIntervalMs?: number; concurrency
       logger.error({ err: String(err) }, '[job-worker] stale-job sweep failed');
     }
     try {
-      for (let i = 0; i < concurrency; i++) {
-        const job = await claimNextQueuedJob();
-        if (!job) break;
-        void dispatchClaimedJob(job); // not awaited — runs concurrently while the loop keeps polling
+      const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: 'queue.paused' } });
+      if (pauseRow?.value === 'true') {
+        logger.info('[job-worker] queue is paused — skipping claim');
+      } else {
+        for (let i = 0; i < concurrency; i++) {
+          const job = await claimNextQueuedJob();
+          if (!job) break;
+          void dispatchClaimedJob(job); // not awaited — runs concurrently while the loop keeps polling
+        }
       }
     } catch (err) {
       logger.error({ err: String(err) }, '[job-worker] claim loop failed');
@@ -390,7 +405,7 @@ export function stopJobWorkerLoop(): void {
 /** Create a job record that tracks a synchronous operation (telemetry only, no background processor). */
 export async function createTrackedJob(
   type: string,
-  config: { tenantId: string; projectId: string; [k: string]: any }
+  config: { tenantId: string; projectId: string;[k: string]: any }
 ): Promise<string> {
   const name = makeJobName(type);
   const job = await withTenant(config.tenantId, (tx) =>
@@ -414,7 +429,7 @@ export async function finishTrackedJob(tenantId: string, jobId: string, error?:
         ...(error ? { error, nextRetryAt } : { progress: 100 }),
       },
     })
-  ).catch(() => {});
+  ).catch(() => { });
 }
 
 /** Enqueue a generic platform job — returns the job ID immediately. */
@@ -691,11 +706,11 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     // Build runtime credentials: convert stored totpSecret → getOneTimeCode function
     const runtimeCreds = config.credentials
       ? {
-          ...config.credentials,
-          ...(config.credentials.totpSecret
-            ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) }
-            : {}),
-        }
+        ...config.credentials,
+        ...(config.credentials.totpSecret
+          ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) }
+          : {}),
+      }
       : undefined;
 
     const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
@@ -997,11 +1012,11 @@ export async function aggregateJobTokens(tenantId: string, jobId: string, projec
       await tx.platformJob.update({
         where: { id: jobId },
         data: {
-          totalPromptTokens:     row.totalPromptTokens,
+          totalPromptTokens: row.totalPromptTokens,
           totalCompletionTokens: row.totalCompletionTokens,
-          totalCacheReadTokens:  row.totalCacheReadTokens,
+          totalCacheReadTokens: row.totalCacheReadTokens,
           totalEstimatedCostUsd: Number(row.totalEstimatedCostUsd),
-          llmCallCount:          row.llmCallCount,
+          llmCallCount: row.llmCallCount,
         },
       });
     });
@@ -1180,10 +1195,10 @@ async function processFigmaImportJob(jobId: string, config: any) {
           const figmaElements = figma.extractElements((frame as any).node);
           for (const el of figmaElements) {
             const role = el.type === 'COMPONENT' || el.type === 'COMPONENT_SET' ? 'COMPONENT'
-                       : el.type === 'INSTANCE' ? 'COMPONENT'
-                       : el.type === 'TEXT' ? 'TEXT'
-                       : el.type === 'FRAME' || el.type === 'GROUP' ? 'REGION'
-                       : el.type;
+              : el.type === 'INSTANCE' ? 'COMPONENT'
+                : el.type === 'TEXT' ? 'TEXT'
+                  : el.type === 'FRAME' || el.type === 'GROUP' ? 'REGION'
+                    : el.type;
             const boundingRect = el.boundingBox
               ? { x: Math.round(el.boundingBox.x), y: Math.round(el.boundingBox.y), width: Math.round(el.boundingBox.width), height: Math.round(el.boundingBox.height) }
               : undefined;
@@ -1261,7 +1276,7 @@ async function processNativeCrawlJob(jobId: string, config: any) {
           deviceFarm,
         },
       })
-    ).catch(() => {});
+    ).catch(() => { });
 
     const workspaceDeviceFarm = await AppBrain.getTenantDeviceFarmConfig(tenantId).catch(() => null);
 
@@ -1328,7 +1343,7 @@ async function processNativeCrawlJob(jobId: string, config: any) {
   } catch (err: any) {
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }
 
@@ -1387,7 +1402,7 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
     try {
       const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
       const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
-      if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => {});
+      if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => { });
       const json = text.match(/\{[\s\S]*\}/)?.[0];
       if (json) parsed = JSON.parse(json);
     } catch (e: any) {
@@ -1396,8 +1411,8 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
 
     await updateJobProgress(tenantId, jobId, 70, `Saving ${parsed.cases?.length ?? 0} test cases`);
 
-    const VALID_KINDS = new Set(['POSITIVE','NEGATIVE','EDGE','STATE_TRANSITION','ACCESSIBILITY','SECURITY','INTEGRATION','E2E','API','EXPLORATORY','UAT','PERFORMANCE','COMPATIBILITY','L10N','USABILITY','LOAD','UNIT','MUTATION','AB_TEST','ALPHA_BETA','SYSTEM','SMOKE','REGRESSION','SANITY']);
-    const VALID_PRIORITIES = new Set(['CRITICAL','HIGH','MEDIUM','LOW']);
+    const VALID_KINDS = new Set(['POSITIVE', 'NEGATIVE', 'EDGE', 'STATE_TRANSITION', 'ACCESSIBILITY', 'SECURITY', 'INTEGRATION', 'E2E', 'API', 'EXPLORATORY', 'UAT', 'PERFORMANCE', 'COMPATIBILITY', 'L10N', 'USABILITY', 'LOAD', 'UNIT', 'MUTATION', 'AB_TEST', 'ALPHA_BETA', 'SYSTEM', 'SMOKE', 'REGRESSION', 'SANITY']);
+    const VALID_PRIORITIES = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']);
     const normalized = (parsed.cases ?? []).map((c: any) => ({
       ref: c.reqRef ?? null,
       traceTo: c.reqRef ?? null,
@@ -1419,7 +1434,7 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
     await Promise.all(
       reqs
         .filter((r: any) => coveredReqIds.has(r.reqId))
-        .map((r: any) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: 'covered' }).catch(() => {}))
+        .map((r: any) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: 'covered' }).catch(() => { }))
     );
 
     await withTenant(tenantId, (tx) =>
@@ -1433,7 +1448,7 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
     logger.error({ jobId, err: msg }, 'processGenerateTestsFromReqsJob failed');
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }
 
@@ -1544,7 +1559,7 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
               } : {}),
             });
           }
-          await AppBrain.recordFlakinessOutcome(tenantId, tc.id).catch(() => {});
+          await AppBrain.recordFlakinessOutcome(tenantId, tc.id).catch(() => { });
         } catch (err: any) {
           failed++;
           runResults.push({ testCaseId: tc.id, status: 'ERROR' });
@@ -1553,7 +1568,7 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
             testCaseId: tc.id,
             status: 'ERROR',
             error: String(err?.message ?? err).slice(0, 500),
-          }).catch(() => {});
+          }).catch(() => { });
         }
       }));
       await updateJobProgress(
@@ -1592,10 +1607,10 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
   } catch (err: any) {
     const msg = String(err?.message ?? err).slice(0, 500);
     logger.error({ jobId, testRunId, err: msg }, 'processRunTestSuiteJob failed');
-    await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => {});
+    await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => { });
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }
 
@@ -1840,12 +1855,12 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
       await onProgress(52, 'Authenticated spider…');
       const runtimeCreds = crawlCfg.username
         ? {
-            username: crawlCfg.username,
-            password: crawlCfg.password ?? '',
-            ...(crawlCfg.totpSecret
-              ? { getOneTimeCode: async () => generateTOTP(crawlCfg.totpSecret!) }
-              : {}),
-          }
+          username: crawlCfg.username,
+          password: crawlCfg.password ?? '',
+          ...(crawlCfg.totpSecret
+            ? { getOneTimeCode: async () => generateTOTP(crawlCfg.totpSecret!) }
+            : {}),
+        }
         : undefined;
 
       const authResult = await crawlProject({
@@ -1897,7 +1912,9 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
     }
 
     // Persist discovered URLs to project crawl config
-    const discoveredUrls = Array.from(allUrls);
+    // Filter sentinel strings (_pending_, _auth_wall_, etc.) that may have leaked in
+    // via onUrlsDiscovered callbacks fired before visited-map entries were updated.
+    const discoveredUrls = Array.from(allUrls).filter(u => u.startsWith('http'));
     const existingCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => ({}));
     await AppBrain.saveCrawlConfig(tenantId, projectId, {
       ...(existingCfg as any),
@@ -2164,7 +2181,7 @@ async function processImportTestsJob(jobId: string, config: any) {
                 status: lr.needsReview ? 'PENDING_REVIEW' : undefined,
               },
             })
-          ).catch(() => {}) // non-fatal per-case
+          ).catch(() => { }) // non-fatal per-case
         )
       );
       labeled = labelResults.length;
@@ -2258,7 +2275,7 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
             testCaseId: tc.id,
             styleProfile: conn.styleProfile,
             pageObjectInventory: conn.pageObjectInventory ?? undefined,
-          }).catch(() => {})
+          }).catch(() => { })
         );
       }
     }
@@ -2272,7 +2289,7 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
     logger.error({ jobId, screenId, err: msg }, 'processGenerateScreenTestsJob failed');
     await withTenant(tenantId, (tx: any) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: msg.slice(0, 500), finishedAt: new Date() } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }
 
@@ -2377,10 +2394,35 @@ async function processCaptureStepsJob(jobId: string, config: any) {
     if (!tc) throw new Error('Test case not found');
 
     const rawSteps = Array.isArray(tc.steps) ? tc.steps : [];
-    const stepTexts = rawSteps.map(captureStepText).filter(Boolean);
+    if (rawSteps.length === 0) throw new Error('Test case has no steps to capture');
+
+    // Resolve env tokens from testDataInline so {{fze.env.URL}} → concrete URL before execution plan
+    const _tdi = ((tc as any).testDataInline ?? {}) as Record<string, unknown>;
+    const _tdiStartUrl = typeof _tdi['startUrl'] === 'string' && (_tdi['startUrl'] as string).startsWith('http')
+      ? (_tdi['startUrl'] as string) : '';
+    const _recordedUrls = Array.isArray(_tdi['recordedUrls']) ? (_tdi['recordedUrls'] as unknown[]).map(String) : [];
+    const _recordedStartUrl = _recordedUrls.find((u) => u.startsWith('http')) ?? '';
+
+    const baseUrl = config.baseUrl || _tdiStartUrl || _recordedStartUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || '';
+
+    // Build env token → concrete value map from testDataInline for step text substitution
+    const _envTokenMap: Record<string, string> = {};
+    for (const [k, v] of Object.entries(_tdi)) {
+      if (typeof v === 'string' && v.startsWith('http')) _envTokenMap[k] = v;
+    }
+    if (_recordedStartUrl && !_envTokenMap['startUrl']) _envTokenMap['startUrl'] = _recordedStartUrl;
+
+    // Replace {{fze.env.X}} and [env:X] tokens in step texts with concrete values from testDataInline.
+    // IMPORTANT: only substitute if we have a concrete value for that key.
+    // Fall back to [key] placeholder — never fall back to baseUrl (would inject URL into credential fields).
+    const resolveStepText = (text: string): string =>
+      text
+        .replace(/\{\{\s*fze\.env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g, (_m, k) => _envTokenMap[k] ?? `[${k}]`)
+        .replace(/\[env:([A-Za-z_][A-Za-z0-9_]*)\]/g, (_m, k) => _envTokenMap[k] ?? `[${k}]`);
+
+    const stepTexts = rawSteps.map(captureStepText).filter(Boolean).map(resolveStepText);
     if (stepTexts.length === 0) throw new Error('Test case has no steps to capture');
 
-    const baseUrl = config.baseUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || '';
     let plan = await generateExecutionPlan(tenantId, { ...tc, steps: stepTexts }, baseUrl).catch(() => []);
     plan = Array.isArray(plan)
       ? plan.filter(Boolean).map((step: any, idx: number) => ({ ...step, stepNumber: idx + 1 }))
@@ -2424,7 +2466,7 @@ async function processCaptureStepsJob(jobId: string, config: any) {
             heartbeatAt: new Date(),
           },
         })
-      ).catch(() => {});
+      ).catch(() => { });
 
       try {
         const heal = await executeExecutionStep(page, step, aiHealingEnabled ? { healCtx: { tenantId, projectId } } : {});
@@ -2433,7 +2475,7 @@ async function processCaptureStepsJob(jobId: string, config: any) {
           result.healTier = heal.healedVia;
           result.healReasoning = heal.healReasoning;
         }
-        await page.waitForLoadState('networkidle', { timeout: parseInt(process.env.TEST_NETWORK_IDLE_TIMEOUT_MS ?? '2000', 10) }).catch(() => {});
+        await page.waitForLoadState('networkidle', { timeout: parseInt(process.env.TEST_NETWORK_IDLE_TIMEOUT_MS ?? '2000', 10) }).catch(() => { });
         result.status = result.healed ? 'HEALED' : 'PASSED';
       } catch (err: any) {
         result.status = step.optional ? 'SKIPPED' : 'FAILED';
@@ -2444,7 +2486,7 @@ async function processCaptureStepsJob(jobId: string, config: any) {
       result.url = currentUrl;
       const ssFilename = `${testCaseId}-capture-step-${stepNumber}${result.status === 'FAILED' ? '-err' : ''}.png`;
       const ssPath = path.join(screenshotDir, ssFilename);
-      await page.screenshot({ path: ssPath, fullPage: false }).catch(() => {});
+      await page.screenshot({ path: ssPath, fullPage: false }).catch(() => { });
       result.screenshotPath = ssPath;
 
       let cloudUrl: string | undefined;
@@ -2466,13 +2508,13 @@ async function processCaptureStepsJob(jobId: string, config: any) {
         const screen = await AppBrain.saveScreen(tenantId, projectId, screenName, currentUrl, jobId, i === 0 ? 'ENTRY_POINT' : 'LINK_FOLLOW').catch(() => null) as any;
         if (screen?.id) {
           result.screenId = screen.id;
-          await AppBrain.saveScreenshot(tenantId, projectId, screen.id, ssPath, screen.version ?? 1, 'DESKTOP', cloudUrl).catch(() => {});
+          await AppBrain.saveScreenshot(tenantId, projectId, screen.id, ssPath, screen.version ?? 1, 'DESKTOP', cloudUrl).catch(() => { });
           for (const el of elements) {
             await AppBrain.saveElement(
               tenantId, projectId, screen.id,
               el.meaning, el.role, el.expectedData, el.notes,
               undefined, true, el.boundingRect, el.ariaState, el.parentLandmark,
-            ).catch(() => {});
+            ).catch(() => { });
           }
           for (const loc of elementsToPlaywrightLocators(elements as any)) {
             if (seenLocatorExpressions.has(loc.expression)) continue;
@@ -2491,7 +2533,7 @@ async function processCaptureStepsJob(jobId: string, config: any) {
     }
 
     if (locators.length > 0) {
-      await AppBrain.saveLocatorsJson(tenantId, testCaseId, locators as any).catch(() => {});
+      await AppBrain.saveLocatorsJson(tenantId, testCaseId, locators as any).catch(() => { });
     }
 
     const firstScreen = screens[0];
@@ -2503,7 +2545,7 @@ async function processCaptureStepsJob(jobId: string, config: any) {
           captureStepScreenshots: stepResults as any,
         },
       })
-    ).catch(() => {});
+    ).catch(() => { });
 
     const finalResult = {
       stepResults,
@@ -2522,11 +2564,11 @@ async function processCaptureStepsJob(jobId: string, config: any) {
     const error = String(err?.message ?? err);
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error, finishedAt: new Date(), heartbeatAt: new Date() } })
-    ).catch(() => {});
+    ).catch(() => { });
     logger.error({ jobId, testCaseId, err: error }, 'processCaptureStepsJob failed');
     throw err;
   } finally {
-    await browser?.close?.().catch(() => {});
+    await browser?.close?.().catch(() => { });
   }
 }
 
@@ -2562,28 +2604,28 @@ jobDispatchTable = {
     await prismaAdmin.platformJob.update({
       where: { id: jobId },
       data: { status: 'FAILED', error: 'Test generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
-    }).catch(() => {});
+    }).catch(() => { });
   },
   GENERATE_DATA: async (jobId: string, _config: any) => {
     logger.error({ jobId }, '[job-worker] GENERATE_DATA claimed by worker — API did not process it in-process');
     await prismaAdmin.platformJob.update({
       where: { id: jobId },
       data: { status: 'FAILED', error: 'Data generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
-    }).catch(() => {});
+    }).catch(() => { });
   },
   GENERATE_SCRIPT: async (jobId: string, _config: any) => {
     logger.error({ jobId }, '[job-worker] GENERATE_SCRIPT claimed by worker — API did not process it in-process');
     await prismaAdmin.platformJob.update({
       where: { id: jobId },
       data: { status: 'FAILED', error: 'Script generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
-    }).catch(() => {});
+    }).catch(() => { });
   },
   GENERATE_TEST_PLAN: async (jobId: string, _config: any) => {
     logger.error({ jobId }, '[job-worker] GENERATE_TEST_PLAN claimed by worker — API did not process it in-process');
     await prismaAdmin.platformJob.update({
       where: { id: jobId },
       data: { status: 'FAILED', error: 'Test plan generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
-    }).catch(() => {});
+    }).catch(() => { });
   },
 };
 
@@ -2880,14 +2922,14 @@ async function processRepoScanJob(jobId: string, config: any) {
         const testPathSet = new Set(testPaths);
         const sourceFilePaths = (language === 'typescript' || language === 'javascript')
           ? treeResult.paths
-              .filter((p: string) =>
-                (p.endsWith('.ts') || p.endsWith('.js')) &&
-                !testPathSet.has(p) &&
-                !p.endsWith('.d.ts') &&
-                !p.includes('node_modules/') &&
-                !p.includes('dist/') &&
-                !p.includes('.config.'))
-              .slice(0, 60)
+            .filter((p: string) =>
+              (p.endsWith('.ts') || p.endsWith('.js')) &&
+              !testPathSet.has(p) &&
+              !p.endsWith('.d.ts') &&
+              !p.includes('node_modules/') &&
+              !p.includes('dist/') &&
+              !p.includes('.config.'))
+            .slice(0, 60)
           : [];
         const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
         // Pass full tree paths separately so detectDataDirs can find testdata/*.json, fixtures/*.yaml etc.
@@ -3019,10 +3061,10 @@ async function processRepoScanJob(jobId: string, config: any) {
     logger.error({ jobId, repoConnectionId, err: msg }, '[repo-scan] failed');
     await withTenant(tenantId, (tx) =>
       tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: 'FAILED', scanError: msg.slice(0, 500) } })
-    ).catch(() => {});
+    ).catch(() => { });
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }
 
@@ -3147,7 +3189,7 @@ async function processBatchGenerateAlignedJob(jobId: string, config: any) {
     logger.error({ jobId, err: msg }, '[batch-generate-aligned] failed');
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }
 
@@ -3276,7 +3318,7 @@ async function processCreatePRJob(jobId: string, config: any) {
     logger.error({ jobId, err: msg }, '[create-pr] failed');
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }
 
@@ -3380,6 +3422,6 @@ async function processNewPagesPipelineJob(jobId: string, config: any) {
     logger.error({ jobId, err: msg }, '[new-pages-pipeline] failed');
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => {});
+    ).catch(() => { });
   }
 }

package/types/job-queue.d.ts

@@ -78,7 +78,9 @@ type ClaimedJob = {
 /** Atomically claims one specific QUEUED job for this worker. Returns null if it was already claimed. */
 export declare function claimSpecificJob(jobId: string): Promise<ClaimedJob | null>;
 /** Atomically claims the oldest QUEUED job across all tenants. SKIP LOCKED makes this
- *  race-safe against any number of other pollers doing the same thing concurrently. */
+ *  race-safe against any number of other pollers doing the same thing concurrently.
+ *  DISCOVER jobs are subject to MAX_CONCURRENT_DISCOVER — skipped when that many are
+ *  already RUNNING to prevent Playwright/DB pool exhaustion from simultaneous crawls. */
 export declare function claimNextQueuedJob(): Promise<ClaimedJob | null>;
 /** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
  *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and