@detiq/crawler 1.1.21 → 1.1.23

package/dist/job-queue.js

@@ -85,6 +85,7 @@ const WORKER_ID = `${os.hostname()}:${process.pid}`;
 const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
 const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15e3;
 const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
+const MAX_CONCURRENT_DISCOVER = Number(process.env.MAX_CONCURRENT_DISCOVER) || 2;
 const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15e3;
 const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 5;
 let jobDispatchTable = null;
@@ -98,12 +99,17 @@ async function claimSpecificJob(jobId) {
   return rows[0] ?? null;
 }
 async function claimNextQueuedJob() {
+  const maxDiscover = MAX_CONCURRENT_DISCOVER;
   const rows = await prismaAdmin.$queryRaw`
     UPDATE "PlatformJob"
     SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
     WHERE id = (
       SELECT id FROM "PlatformJob"
       WHERE status = 'QUEUED'
+        AND (
+          type != 'DISCOVER'
+          OR (SELECT COUNT(*) FROM "PlatformJob" WHERE status = 'RUNNING' AND type = 'DISCOVER') < ${maxDiscover}
+        )
       ORDER BY "queuedAt" ASC
       FOR UPDATE SKIP LOCKED
       LIMIT 1
@@ -206,10 +212,15 @@ function startJobWorkerLoop(opts) {
       logger.error({ err: String(err) }, "[job-worker] stale-job sweep failed");
     }
     try {
-      for (let i = 0; i < concurrency; i++) {
-        const job = await claimNextQueuedJob();
-        if (!job) break;
-        void dispatchClaimedJob(job);
+      const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: "queue.paused" } });
+      if (pauseRow?.value === "true") {
+        logger.info("[job-worker] queue is paused \u2014 skipping claim");
+      } else {
+        for (let i = 0; i < concurrency; i++) {
+          const job = await claimNextQueuedJob();
+          if (!job) break;
+          void dispatchClaimedJob(job);
+        }
       }
     } catch (err) {
       logger.error({ err: String(err) }, "[job-worker] claim loop failed");
@@ -1530,7 +1541,7 @@ async function processDiscoverJob(jobId, config, opts) {
       }
       logger.info({ count: allUrls.size }, "[discover] Phase 3 auth spider done");
     }
-    const discoveredUrls = Array.from(allUrls);
+    const discoveredUrls = Array.from(allUrls).filter((u) => u.startsWith("http"));
     const existingCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => ({}));
     await AppBrain.saveCrawlConfig(tenantId, projectId, {
       ...existingCfg,
@@ -1941,9 +1952,20 @@ async function processCaptureStepsJob(jobId, config) {
     ]);
     if (!tc) throw new Error("Test case not found");
     const rawSteps = Array.isArray(tc.steps) ? tc.steps : [];
-    const stepTexts = rawSteps.map(captureStepText).filter(Boolean);
+    if (rawSteps.length === 0) throw new Error("Test case has no steps to capture");
+    const _tdi = tc.testDataInline ?? {};
+    const _tdiStartUrl = typeof _tdi["startUrl"] === "string" && _tdi["startUrl"].startsWith("http") ? _tdi["startUrl"] : "";
+    const _recordedUrls = Array.isArray(_tdi["recordedUrls"]) ? _tdi["recordedUrls"].map(String) : [];
+    const _recordedStartUrl = _recordedUrls.find((u) => u.startsWith("http")) ?? "";
+    const baseUrl = config.baseUrl || _tdiStartUrl || _recordedStartUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || "";
+    const _envTokenMap = {};
+    for (const [k, v] of Object.entries(_tdi)) {
+      if (typeof v === "string" && v.startsWith("http")) _envTokenMap[k] = v;
+    }
+    if (_recordedStartUrl && !_envTokenMap["startUrl"]) _envTokenMap["startUrl"] = _recordedStartUrl;
+    const resolveStepText = (text) => text.replace(/\{\{\s*fze\.env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g, (_m, k) => _envTokenMap[k] ?? `[${k}]`).replace(/\[env:([A-Za-z_][A-Za-z0-9_]*)\]/g, (_m, k) => _envTokenMap[k] ?? `[${k}]`);
+    const stepTexts = rawSteps.map(captureStepText).filter(Boolean).map(resolveStepText);
     if (stepTexts.length === 0) throw new Error("Test case has no steps to capture");
-    const baseUrl = config.baseUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || "";
     let plan = await generateExecutionPlan(tenantId, { ...tc, steps: stepTexts }, baseUrl).catch(() => []);
     plan = Array.isArray(plan) ? plan.filter(Boolean).map((step, idx) => ({ ...step, stepNumber: idx + 1 })) : [];
     if (plan.length === 0) plan = fallbackExecutionPlan(rawSteps);