@detiq/crawler 1.1.22 → 1.1.24

package/dist/job-queue.js

@@ -194,12 +194,53 @@ async function dispatchClaimedJob(job) {
     logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, "[job-worker] job threw uncaught error");
     await prismaAdmin.platformJob.update({
       where: { id: job.id },
-      data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() }
+      data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date(), nextRetryAt: new Date(Date.now() + 6e4) }
     }).catch(() => {
     });
   }
 }
 let workerLoopTimer = null;
+async function retryEligibleFailedJobs() {
+  const rows = await prismaAdmin.$queryRaw`
+    UPDATE "PlatformJob"
+    SET status = 'QUEUED',
+        "retryCount" = "retryCount" + 1,
+        "nextRetryAt" = NOW() + (INTERVAL '1 minute' * POWER(2, "retryCount")),
+        "workerId" = NULL
+    WHERE status = 'FAILED'
+      AND dlq = false
+      AND "nextRetryAt" <= NOW()
+      AND "retryCount" < CASE
+        WHEN type IN ('CRAWL', 'NATIVE_CRAWL') THEN 3
+        WHEN type IN ('GENERATE_TESTS', 'IMPORT_TESTS', 'RUN_TEST_SUITE', 'VISUAL_REGRESSION') THEN 2
+        WHEN type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TEST_PLAN') THEN 0
+        ELSE 1
+      END
+    RETURNING id
+  `;
+  if (rows.length > 0) {
+    logger.info({ jobIds: rows.map((r) => r.id) }, "[job-worker] requeued failed job(s) for retry");
+  }
+}
+async function promoteToDeadLetterQueue() {
+  const rows = await prismaAdmin.$queryRaw`
+    UPDATE "PlatformJob"
+    SET dlq = true,
+        "dlqReason" = error
+    WHERE status = 'FAILED'
+      AND dlq = false
+      AND "retryCount" >= CASE
+        WHEN type IN ('CRAWL', 'NATIVE_CRAWL') THEN 3
+        WHEN type IN ('GENERATE_TESTS', 'IMPORT_TESTS', 'RUN_TEST_SUITE', 'VISUAL_REGRESSION') THEN 2
+        WHEN type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TEST_PLAN') THEN 0
+        ELSE 1
+      END
+    RETURNING id
+  `;
+  if (rows.length > 0) {
+    logger.error({ jobIds: rows.map((r) => r.id) }, "[job-worker] promoted failed job(s) to DLQ");
+  }
+}
 function startJobWorkerLoop(opts) {
   if (workerLoopTimer) return;
   const pollIntervalMs = opts?.pollIntervalMs ?? WORKER_POLL_INTERVAL_MS;
@@ -207,15 +248,22 @@ function startJobWorkerLoop(opts) {
   logger.info({ workerId: WORKER_ID, pollIntervalMs, concurrency }, "[job-worker] starting");
   const tick = async () => {
     try {
+      await retryEligibleFailedJobs();
+      await promoteToDeadLetterQueue();
       await requeueStaleRunningJobs();
     } catch (err) {
-      logger.error({ err: String(err) }, "[job-worker] stale-job sweep failed");
+      logger.error({ err: String(err) }, "[job-worker] stale/retry job sweep failed");
     }
     try {
-      for (let i = 0; i < concurrency; i++) {
-        const job = await claimNextQueuedJob();
-        if (!job) break;
-        void dispatchClaimedJob(job);
+      const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: "queue.paused" } });
+      if (pauseRow?.value === "true") {
+        logger.info("[job-worker] queue is paused \u2014 skipping claim");
+      } else {
+        for (let i = 0; i < concurrency; i++) {
+          const job = await claimNextQueuedJob();
+          if (!job) break;
+          void dispatchClaimedJob(job);
+        }
       }
     } catch (err) {
       logger.error({ err: String(err) }, "[job-worker] claim loop failed");
@@ -272,9 +320,9 @@ async function enqueueJob(type, config) {
       }
     })
   );
-  if (type === "CRAWL" && process.env.JOB_QUEUE_BACKEND === "sqs") {
+  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
     const { sendJobToSqs } = await import("./sqs-queue.js");
-    await sendJobToSqs(job.id, tenantId, config);
+    await sendJobToSqs(job.id);
   } else if (process.env.DISABLE_WORKER !== "true") {
     setImmediate(() => claimSpecificJob(job.id).then((claimed) => claimed && dispatchClaimedJob(claimed)));
   }
@@ -656,8 +704,8 @@ async function processJob(jobId, config, opts) {
                 WHERE tc."tenantId" = ${tenantId}
                   AND s."projectId" = ${projectId}
                   AND s."elementHashChangedAt" IS NOT NULL
-                  AND tc."genAt" IS NOT NULL
-                  AND s."elementHashChangedAt" > tc."genAt"
+                  AND tc."createdAt" IS NOT NULL
+                  AND s."elementHashChangedAt" > tc."createdAt"
                   AND tc.status != 'PENDING_REVIEW'
                   AND tc.status != 'RETIRED'
               `;
@@ -706,6 +754,7 @@ async function processJob(jobId, config, opts) {
           status: aborted ? "CANCELLED" : "FAILED",
           error: aborted ? "Stopped by user" : String(err),
           finishedAt: /* @__PURE__ */ new Date(),
+          ...aborted ? {} : { nextRetryAt: new Date(Date.now() + 6e4) },
           ...checkpoint ? { result: { checkpoint } } : {}
         }
       })
@@ -788,7 +837,7 @@ async function processApiTestsJob(jobId, config) {
   } catch (err) {
     await withTenant(
       tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
+      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date(), nextRetryAt: new Date(Date.now() + 6e4) } })
     );
   }
 }
@@ -823,7 +872,7 @@ async function processSecurityTestsJob(jobId, config) {
   } catch (err) {
     await withTenant(
       tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
+      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date(), nextRetryAt: new Date(Date.now() + 6e4) } })
     );
   }
 }
@@ -858,7 +907,7 @@ async function processIntegrationTestsJob(jobId, config) {
   } catch (err) {
     await withTenant(
       tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
+      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date(), nextRetryAt: new Date(Date.now() + 6e4) } })
     );
   }
 }
@@ -1045,8 +1094,8 @@ async function processNativeCrawlJob(jobId, config) {
                 WHERE tc."tenantId" = ${tenantId}
                   AND s."projectId" = ${projectId}
                   AND s."elementHashChangedAt" IS NOT NULL
-                  AND tc."genAt" IS NOT NULL
-                  AND s."elementHashChangedAt" > tc."genAt"
+                  AND tc."createdAt" IS NOT NULL
+                  AND s."elementHashChangedAt" > tc."createdAt"
                   AND tc.status != 'PENDING_REVIEW'
                   AND tc.status != 'RETIRED'
               `;
@@ -1492,43 +1541,48 @@ async function processDiscoverJob(jobId, config, opts) {
         password: crawlCfg.password ?? "",
         ...crawlCfg.totpSecret ? { getOneTimeCode: async () => generateTOTP(crawlCfg.totpSecret) } : {}
       } : void 0;
-      const authResult = await crawlProject({
-        tenantId,
-        projectId,
-        appUrl,
-        discoveryMode: true,
-        credentials: runtimeCreds,
-        storageState: crawlCfg.storageState,
-        httpBasicUsername: crawlCfg.httpBasicUsername,
-        httpBasicPassword: crawlCfg.httpBasicPassword,
-        authToken: crawlCfg.authToken,
-        authTokenHeader: crawlCfg.authTokenHeader,
-        authTokenPrefix: crawlCfg.authTokenPrefix,
-        startUrls: crawlCfg.postLoginUrl ? [crawlCfg.postLoginUrl, ...sitemapUrls.length > 0 ? sitemapUrls : []] : sitemapUrls.length > 0 ? sitemapUrls : void 0,
-        screenshotDir,
-        maxScreens: maxDiscoverUrls,
-        proxyUrl,
-        chromeProfilePath: crawlCfg.chromeProfilePath,
-        useFirefox: crawlCfg.useFirefox,
-        storageConfig: discoverStorageConfig,
-        captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
-        captchaSolverProvider: crawlCfg.captchaSolverProvider,
-        mailslurpApiKey: crawlCfg.mailslurpApiKey,
-        mailslurpInboxId: crawlCfg.mailslurpInboxId,
-        loginInstructions: crawlCfg.loginInstructions,
-        ...resolvedSso,
-        abortSignal: controller.signal,
-        jobId,
-        onUrlsDiscovered: async (urls) => {
-          for (const u of urls) allUrls.add(u);
-          const pct = 52 + Math.round(allUrls.size / Math.max(1, maxDiscoverUrls) * 38);
-          await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)\u2026`);
-        }
-      }, async (pct, step) => {
-        await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
-      }).catch((err) => {
+      const PHASE3_TIMEOUT_MS = 20 * 6e4;
+      const authResult = await Promise.race([
+        crawlProject({
+          tenantId,
+          projectId,
+          appUrl,
+          discoveryMode: true,
+          credentials: runtimeCreds,
+          storageState: crawlCfg.storageState,
+          httpBasicUsername: crawlCfg.httpBasicUsername,
+          httpBasicPassword: crawlCfg.httpBasicPassword,
+          authToken: crawlCfg.authToken,
+          authTokenHeader: crawlCfg.authTokenHeader,
+          authTokenPrefix: crawlCfg.authTokenPrefix,
+          startUrls: crawlCfg.postLoginUrl ? [crawlCfg.postLoginUrl, ...sitemapUrls.length > 0 ? sitemapUrls : []] : sitemapUrls.length > 0 ? sitemapUrls : void 0,
+          screenshotDir,
+          maxScreens: maxDiscoverUrls,
+          proxyUrl,
+          chromeProfilePath: crawlCfg.chromeProfilePath,
+          useFirefox: crawlCfg.useFirefox,
+          storageConfig: discoverStorageConfig,
+          captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
+          captchaSolverProvider: crawlCfg.captchaSolverProvider,
+          mailslurpApiKey: crawlCfg.mailslurpApiKey,
+          mailslurpInboxId: crawlCfg.mailslurpInboxId,
+          loginInstructions: crawlCfg.loginInstructions,
+          ...resolvedSso,
+          abortSignal: controller.signal,
+          jobId,
+          onUrlsDiscovered: async (urls) => {
+            for (const u of urls) allUrls.add(u);
+            const pct = 52 + Math.round(allUrls.size / Math.max(1, maxDiscoverUrls) * 38);
+            await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)\u2026`);
+          }
+        }, async (pct, step) => {
+          await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
+        }),
+        new Promise((_, rej) => setTimeout(() => rej(new Error("AUTH_PHASE_TIMEOUT")), PHASE3_TIMEOUT_MS))
+      ]).catch((err) => {
         if (err?.message === "CRAWL_ABORTED") throw err;
-        logger.warn({ err: err?.message }, "[discover] Phase 3 authenticated spider failed \u2014 using unauthenticated results");
+        const reason = err?.message === "AUTH_PHASE_TIMEOUT" ? "Authenticated spider timed out after 20 minutes" : err?.message;
+        logger.warn({ reason }, "[discover] Phase 3 authenticated spider failed \u2014 using unauthenticated results");
         return { discoveredUrls: [] };
       });
       for (const u of authResult.discoveredUrls ?? []) {
@@ -1947,9 +2001,20 @@ async function processCaptureStepsJob(jobId, config) {
     ]);
     if (!tc) throw new Error("Test case not found");
     const rawSteps = Array.isArray(tc.steps) ? tc.steps : [];
-    const stepTexts = rawSteps.map(captureStepText).filter(Boolean);
+    if (rawSteps.length === 0) throw new Error("Test case has no steps to capture");
+    const _tdi = tc.testDataInline ?? {};
+    const _tdiStartUrl = typeof _tdi["startUrl"] === "string" && _tdi["startUrl"].startsWith("http") ? _tdi["startUrl"] : "";
+    const _recordedUrls = Array.isArray(_tdi["recordedUrls"]) ? _tdi["recordedUrls"].map(String) : [];
+    const _recordedStartUrl = _recordedUrls.find((u) => u.startsWith("http")) ?? "";
+    const baseUrl = config.baseUrl || _tdiStartUrl || _recordedStartUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || "";
+    const _envTokenMap = {};
+    for (const [k, v] of Object.entries(_tdi)) {
+      if (typeof v === "string" && v.startsWith("http")) _envTokenMap[k] = v;
+    }
+    if (_recordedStartUrl && !_envTokenMap["startUrl"]) _envTokenMap["startUrl"] = _recordedStartUrl;
+    const resolveStepText = (text) => text.replace(/\{\{\s*fze\.env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g, (_m, k) => _envTokenMap[k] ?? `[${k}]`).replace(/\[env:([A-Za-z_][A-Za-z0-9_]*)\]/g, (_m, k) => _envTokenMap[k] ?? `[${k}]`);
+    const stepTexts = rawSteps.map(captureStepText).filter(Boolean).map(resolveStepText);
     if (stepTexts.length === 0) throw new Error("Test case has no steps to capture");
-    const baseUrl = config.baseUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || "";
     let plan = await generateExecutionPlan(tenantId, { ...tc, steps: stepTexts }, baseUrl).catch(() => []);
     plan = Array.isArray(plan) ? plan.filter(Boolean).map((step, idx) => ({ ...step, stepNumber: idx + 1 })) : [];
     if (plan.length === 0) plan = fallbackExecutionPlan(rawSteps);
@@ -2106,6 +2171,283 @@ async function processCaptureStepsJob(jobId, config) {
     });
   }
 }
+async function processMultiRoleCrawlJob(jobId, config) {
+  const { tenantId, projectId } = config;
+  try {
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.updateMany({
+        where: { id: jobId, status: { not: "CANCELLED" } },
+        data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date(), heartbeatAt: /* @__PURE__ */ new Date(), workerId: WORKER_ID }
+      })
+    );
+    await updateJobProgress(tenantId, jobId, 5, "Loading project crawl config\u2026");
+    const proj = await withTenant(
+      tenantId,
+      (tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
+    );
+    const savedCrawlConfig = proj?.crawlConfig ?? {};
+    const credentialProfiles = config.credentialProfiles ?? [];
+    await updateJobProgress(tenantId, jobId, 10, "Enqueuing primary crawl\u2026");
+    const primaryJobId = await enqueueJob("CRAWL", {
+      tenantId,
+      projectId,
+      appUrl: config.appUrl || savedCrawlConfig.appUrl,
+      credentials: savedCrawlConfig.username && savedCrawlConfig.password ? { username: savedCrawlConfig.username, password: savedCrawlConfig.password, ...savedCrawlConfig.totpSecret ? { totpSecret: savedCrawlConfig.totpSecret } : {} } : void 0,
+      storageState: savedCrawlConfig.storageState,
+      screenshotDir: platformConfig.screenshotDir,
+      startUrls: savedCrawlConfig.seedUrls?.length ? savedCrawlConfig.seedUrls : void 0,
+      deviceProfiles: savedCrawlConfig.deviceProfiles ?? ["DESKTOP"],
+      loginInstructions: savedCrawlConfig.loginInstructions,
+      proxyUrl: savedCrawlConfig.proxyUrl,
+      useSitemap: true,
+      sitemapUrl: savedCrawlConfig.sitemapUrl
+    });
+    logger.info({ primaryJobId, projectId }, "[multi-role-crawl] primary CRAWL enqueued");
+    const PRIMARY_TIMEOUT_MS = 2 * 60 * 60 * 1e3;
+    const POLL_INTERVAL_MS = 1e4;
+    const deadline = Date.now() + PRIMARY_TIMEOUT_MS;
+    let primaryStatus = "QUEUED";
+    while (Date.now() < deadline) {
+      const primaryJob = await prismaAdmin.platformJob.findUnique({
+        where: { id: primaryJobId },
+        select: { status: true }
+      }).catch(() => null);
+      primaryStatus = primaryJob?.status ?? "UNKNOWN";
+      if (primaryStatus === "DONE" || primaryStatus === "FAILED" || primaryStatus === "CANCELLED") break;
+      await updateJobProgress(tenantId, jobId, 20, `Waiting for primary crawl (${primaryStatus})\u2026`);
+      await new Promise((res) => setTimeout(res, POLL_INTERVAL_MS));
+    }
+    if (primaryStatus !== "DONE") {
+      throw new Error(`Primary crawl ${primaryJobId} ended with status ${primaryStatus} \u2014 aborting role crawls`);
+    }
+    await updateJobProgress(tenantId, jobId, 50, "Collecting screen URLs for role crawls\u2026");
+    const screens = await prismaAdmin.screen.findMany({
+      where: { tenantId, projectId, status: "ACTIVE" },
+      select: { id: true, url: true }
+    }).catch(() => []);
+    const screenUrls = screens.filter((s) => s.url).map((s) => s.url);
+    logger.info({ screenCount: screenUrls.length, projectId }, "[multi-role-crawl] screen URLs collected");
+    if (credentialProfiles.length === 0) {
+      logger.warn({ projectId, jobId }, "[multi-role-crawl] no credential profiles \u2014 nothing to role-crawl");
+    }
+    const roleJobs = [];
+    for (const profile of credentialProfiles) {
+      if (!profile.username || !profile.password) continue;
+      const roleJobId = await enqueueJob("ROLE_CRAWL", {
+        tenantId,
+        projectId,
+        appUrl: config.appUrl || savedCrawlConfig.appUrl,
+        credentials: { username: profile.username, password: profile.password },
+        roleProfileName: profile.name,
+        parentJobId: jobId,
+        selectiveUrls: config.fullReCrawl ? void 0 : screenUrls.length > 0 ? screenUrls : void 0,
+        startUrls: config.fullReCrawl ? savedCrawlConfig.seedUrls ?? [] : void 0,
+        maxScreens: config.fullReCrawl ? void 0 : screenUrls.length + 10,
+        screenshotDir: platformConfig.screenshotDir,
+        storageState: savedCrawlConfig.storageState,
+        deviceProfiles: savedCrawlConfig.deviceProfiles ?? ["DESKTOP"],
+        loginInstructions: savedCrawlConfig.loginInstructions,
+        proxyUrl: savedCrawlConfig.proxyUrl
+      });
+      roleJobs.push({ roleName: profile.name, jobId: roleJobId });
+      logger.info({ roleJobId, roleName: profile.name, projectId }, "[multi-role-crawl] ROLE_CRAWL enqueued");
+    }
+    const ROLE_TIMEOUT_MS = 2 * 60 * 60 * 1e3;
+    const roleDeadline = Date.now() + ROLE_TIMEOUT_MS;
+    const roleResults = [];
+    while (Date.now() < roleDeadline && roleJobs.length > 0) {
+      const pending = [];
+      for (const rj of roleJobs) {
+        const rJob = await prismaAdmin.platformJob.findUnique({
+          where: { id: rj.jobId },
+          select: { status: true, result: true }
+        }).catch(() => null);
+        const st = rJob?.status ?? "UNKNOWN";
+        if (st === "DONE" || st === "FAILED" || st === "CANCELLED") {
+          roleResults.push({
+            roleName: rj.roleName,
+            jobId: rj.jobId,
+            status: st,
+            screensTagged: rJob?.result?.screensTagged
+          });
+        } else {
+          pending.push(rj);
+        }
+      }
+      roleJobs.length = 0;
+      roleJobs.push(...pending);
+      if (roleJobs.length === 0) break;
+      const pct = Math.round(60 + roleResults.length / (roleResults.length + roleJobs.length) * 35);
+      await updateJobProgress(tenantId, jobId, pct, `Role crawls: ${roleResults.length} done, ${roleJobs.length} pending\u2026`);
+      await new Promise((res) => setTimeout(res, POLL_INTERVAL_MS));
+    }
+    for (const rj of roleJobs) {
+      roleResults.push({ roleName: rj.roleName, jobId: rj.jobId, status: "TIMEOUT" });
+    }
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.update({
+        where: { id: jobId },
+        data: {
+          status: "DONE",
+          progress: 100,
+          finishedAt: /* @__PURE__ */ new Date(),
+          result: { primaryJobId, roleJobs: roleResults }
+        }
+      })
+    );
+    logger.info({ jobId, primaryJobId, roleCount: roleResults.length, projectId }, "[multi-role-crawl] completed");
+  } catch (err) {
+    const msg = String(err?.message ?? err);
+    logger.error({ jobId, err: msg }, "[multi-role-crawl] failed");
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: msg, finishedAt: /* @__PURE__ */ new Date() } })
+    ).catch(() => {
+    });
+  }
+}
+async function processRoleCrawlJob(jobId, config) {
+  const { tenantId, projectId } = config;
+  try {
+    const claimed = await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.updateMany({
+        where: { id: jobId, status: { not: "CANCELLED" } },
+        data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date(), heartbeatAt: /* @__PURE__ */ new Date(), workerId: WORKER_ID }
+      })
+    );
+    if (claimed.count === 0) return;
+    const controller = new AbortController();
+    runningControllers.set(jobId, controller);
+    const heartbeatTimer = setInterval(() => void heartbeatJob(tenantId, jobId), JOB_HEARTBEAT_INTERVAL_MS);
+    const onProgress = async (pct, step) => {
+      await updateJobProgress(tenantId, jobId, pct, step);
+    };
+    try {
+      const proj = await withTenant(
+        tenantId,
+        (tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
+      );
+      const savedCrawlConfig = proj?.crawlConfig ?? {};
+      const runtimeCreds = config.credentials ? {
+        ...config.credentials,
+        ...config.credentials.totpSecret ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) } : {}
+      } : void 0;
+      const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
+      const result = await crawlProject({
+        ...config,
+        credentials: runtimeCreds,
+        storageState: config.storageState ?? savedCrawlConfig.storageState,
+        proxyUrl: config.proxyUrl ?? savedCrawlConfig.proxyUrl,
+        loginInstructions: config.loginInstructions ?? savedCrawlConfig.loginInstructions,
+        abortSignal: controller.signal,
+        jobId,
+        storageConfig: storageConfigForCrawl
+      }, onProgress);
+      clearInterval(heartbeatTimer);
+      runningControllers.delete(jobId);
+      const roleName = config.roleProfileName;
+      const discovered = result.discovered ?? [];
+      const roleScreenIds = discovered.filter((d) => d.screenId).map((d) => d.screenId);
+      if (roleScreenIds.length > 0 && roleName) {
+        await prismaAdmin.$executeRaw`
+          UPDATE "Screen"
+          SET "roleAccess" = array(SELECT DISTINCT unnest(array_append("roleAccess", ${roleName}::text)))
+          WHERE id = ANY(${roleScreenIds}::text[])
+            AND "tenantId" = ${tenantId}
+        `.catch((e) => logger.warn({ err: e?.message, roleName, count: roleScreenIds.length }, "[role-crawl] roleAccess tag failed (non-fatal)"));
+      }
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            status: "DONE",
+            progress: 100,
+            finishedAt: /* @__PURE__ */ new Date(),
+            result: { screensTagged: roleScreenIds.length, roleName }
+          }
+        })
+      );
+      logger.info({ jobId, roleName, screensTagged: roleScreenIds.length, projectId }, "[role-crawl] completed");
+    } catch (err) {
+      clearInterval(heartbeatTimer);
+      runningControllers.delete(jobId);
+      const aborted = String(err).includes("CRAWL_ABORTED");
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            status: aborted ? "CANCELLED" : "FAILED",
+            error: aborted ? "Stopped by user" : String(err),
+            finishedAt: /* @__PURE__ */ new Date()
+          }
+        })
+      );
+    } finally {
+      runningControllers.delete(jobId);
+    }
+  } catch (err) {
+    const msg = String(err?.message ?? err);
+    logger.error({ jobId, err: msg }, "[role-crawl] outer failure");
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: msg, finishedAt: /* @__PURE__ */ new Date() } })
+    ).catch(() => {
+    });
+  }
+}
+async function processGenerateScriptJob(jobId, config) {
+  const { tenantId, projectId, language } = config;
+  try {
+    await updateJobProgress(tenantId, jobId, 1, "Loading test cases\u2026").catch(() => {
+    });
+    if (config.packId) {
+      const { generatePackScript } = await import("@detiq/agents");
+      const result = await generatePackScript({ tenantId, projectId, packType: config.packId, language });
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({
+          where: { id: jobId },
+          data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result }
+        })
+      );
+    } else {
+      const { generateSuiteScript } = await import("@detiq/agents");
+      let progressChain = Promise.resolve();
+      const result = await generateSuiteScript({
+        tenantId,
+        projectId,
+        suiteId: config.suiteId,
+        language,
+        caseIds: config.caseIds ?? void 0,
+        useV2: config.useV2,
+        styleProfile: config.repoStyleProfile ?? void 0,
+        pageObjectInventory: config.repoPageObjects ?? void 0,
+        jobId,
+        onProgress: (done, total) => {
+          progressChain = progressChain.then(() => updateJobProgress(tenantId, jobId, Math.floor(done / total * 95), `Generated ${done}/${total} scripts`)).catch(() => {
+          });
+        }
+      });
+      await progressChain.catch(() => {
+      });
+      await import("@detiq/agents").then(({ refreshScriptInventory }) => refreshScriptInventory(tenantId, projectId)).catch((e) => logger.warn({ err: e?.message }, "[generate-script] refreshScriptInventory failed"));
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({
+          where: { id: jobId },
+          data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result }
+        })
+      );
+    }
+  } catch (e) {
+    await finishTrackedJob(tenantId, jobId, e?.message ?? "generate-script failed");
+  }
+}
 jobDispatchTable = {
   CRAWL: processJob,
   CAPTURE_STEPS: processCaptureStepsJob,
@@ -2125,7 +2467,10 @@ jobDispatchTable = {
   BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
   CREATE_PR: processCreatePRJob,
   NEW_PAGES_PIPELINE: processNewPagesPipelineJob,
-  // The following job types are processed in-process by the API (never QUEUED).
+  MULTI_ROLE_CRAWL: processMultiRoleCrawlJob,
+  ROLE_CRAWL: processRoleCrawlJob,
+  GENERATE_SCRIPT: processGenerateScriptJob,
+  // The following job types are still processed in-process by the API.
   // If the worker claims one, the API crashed after creating it — mark failed so it's visible.
   GENERATE_TESTS: async (jobId, _config) => {
     logger.error({ jobId }, "[job-worker] GENERATE_TESTS claimed by worker \u2014 API did not process it in-process");
@@ -2143,14 +2488,6 @@ jobDispatchTable = {
     }).catch(() => {
     });
   },
-  GENERATE_SCRIPT: async (jobId, _config) => {
-    logger.error({ jobId }, "[job-worker] GENERATE_SCRIPT claimed by worker \u2014 API did not process it in-process");
-    await prismaAdmin.platformJob.update({
-      where: { id: jobId },
-      data: { status: "FAILED", error: "Script generation was not completed by the API \u2014 please retry from the UI.", finishedAt: /* @__PURE__ */ new Date() }
-    }).catch(() => {
-    });
-  },
   GENERATE_TEST_PLAN: async (jobId, _config) => {
     logger.error({ jobId }, "[job-worker] GENERATE_TEST_PLAN claimed by worker \u2014 API did not process it in-process");
     await prismaAdmin.platformJob.update({
@@ -2189,6 +2526,13 @@ async function processRepoScanJob(jobId, config) {
     const treeResult = await fetchRepoTree(token, conn.repoFullName, "HEAD");
     if (treeResult.truncated) {
       console.warn(`[repo-scan] ${conn.repoFullName}: tree truncated (>100k blobs) \u2014 detection may be incomplete`);
+      await withTenant(
+        tenantId,
+        (tx) => tx.gitHubRepoConnection.update({
+          where: { id: repoConnectionId },
+          data: { scanError: "Repository tree was truncated by GitHub (>100k files) \u2014 some test files in deep subdirectories may be missing from this scan." }
+        })
+      );
     }
     const frameworks = detectFramework(treeResult.paths);
     const primary = frameworks[0];
@@ -2200,12 +2544,17 @@ async function processRepoScanJob(jobId, config) {
     const { minimatch } = await import("minimatch").catch(() => ({ minimatch: null }));
     const isPythonTestFile = (p) => p.endsWith(".py") && !p.endsWith("__init__.py") && !p.endsWith("conftest.py") && (p.endsWith("_test.py") || /\/test_[^/]+\.py$/.test(p) || /^test_[^/]+\.py$/.test(p) || p.includes("/tests/") || p.includes("/test/"));
     const isJavaTestFile = (p) => p.endsWith(".java") && (/[A-Z]\w*Tests?\.java$/.test(p) || /\/tests?\//.test(p) && /[A-Z][a-z]/.test(p.split("/").pop() ?? ""));
+    const testGlobs = minimatch ? Array.from(/* @__PURE__ */ new Set([
+      testGlob,
+      ...testGlob === "**/*.spec.ts" ? ["**/*.test.ts", "**/*.cy.ts"] : [],
+      ...testGlob === "**/*.spec.js" ? ["**/*.test.js", "**/*.cy.js"] : []
+    ])) : [];
     const testPaths = treeResult.paths.filter((p) => {
       if (language === "java") return isJavaTestFile(p);
       if (language === "python") return isPythonTestFile(p);
-      if (minimatch) return minimatch(p, testGlob, { matchBase: true });
-      return p.endsWith(".spec.ts") || p.endsWith(".spec.js") || p.endsWith(".test.ts") || p.endsWith(".test.js") || p.endsWith("_test.py") || p.includes("/test_");
-    }).slice(0, 500);
+      if (testGlobs.length) return testGlobs.some((g) => minimatch(p, g, { matchBase: true }));
+      return p.endsWith(".spec.ts") || p.endsWith(".spec.js") || p.endsWith(".test.ts") || p.endsWith(".test.js") || p.endsWith(".cy.ts") || p.endsWith(".cy.js") || p.endsWith("_test.py") || p.includes("/test_");
+    }).slice(0, 2e3);
     await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
     if (testPaths.length === 0) {
       logger.warn({ jobId, repoConnectionId, framework }, "[repo-scan] no test files matched glob \u2014 scan complete with 0 results");
@@ -2576,6 +2925,17 @@ async function processRepoScanJob(jobId, config) {
     } catch (e) {
       logger.warn({ jobId, repoConnectionId, err: e?.message }, "[repo-scan] repo skill files generation failed (non-fatal)");
     }
+    try {
+      const { alignSkillFile } = await import("@detiq/agents");
+      await alignSkillFile(tenantId, projectId, {
+        framework,
+        language,
+        styleProfile: resolvedStyleProfile,
+        repoFullName: conn.repoFullName
+      });
+    } catch (e) {
+      logger.warn({ jobId, repoConnectionId, err: e?.message }, "[repo-scan] alignSkillFile failed (non-fatal)");
+    }
     await updateJobProgress(tenantId, jobId, 100, `Done: ${mapped} mapped, ${unmapped} unmapped`);
     await withTenant(
       tenantId,
@@ -2731,7 +3091,7 @@ async function processCreatePRJob(jobId, config) {
       scripts = await withTenant(
         tenantId,
         (tx) => tx.testScript.findMany({
-          where: { id: { in: testScriptIds }, tenantId },
+          where: { id: { in: testScriptIds }, tenantId, code: { not: null } },
           include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } }
         })
       );
@@ -2739,7 +3099,7 @@ async function processCreatePRJob(jobId, config) {
       scripts = await withTenant(
         tenantId,
         (tx) => tx.testScript.findMany({
-          where: { tenantId, projectId, pushedToPR: false, validationStatus: { in: ["PASS", "PENDING"] } },
+          where: { tenantId, projectId, pushedToPR: false, validationStatus: { in: ["PASS", "PENDING"] }, code: { not: null } },
           include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } }
         })
       );
@@ -2925,6 +3285,7 @@ export {
   claimNextQueuedJob,
   claimSpecificJob,
   createTrackedJob,
+  dispatchClaimedJob,
   enqueueJob,
   finishTrackedJob,
   generateTOTP,
@@ -2932,10 +3293,12 @@ export {
   listJobs,
   processDataRetentionPurge,
   processJob,
+  promoteToDeadLetterQueue,
   requeueOrphanedJobs,
   requeueStaleRunningJobs,
   rerunJob,
   resumeJob,
+  retryEligibleFailedJobs,
   startJobWorkerLoop,
   stopJobWorkerLoop,
   updateJobProgress