@detiq/crawler 1.1.23 → 1.1.25

package/src/job-queue.ts

@@ -178,6 +178,40 @@ export interface PlatformJobConfig {
   ssoScope?: string;
   /** HTTP/HTTPS/SOCKS5 proxy URL — routes browser through this proxy for bot-protected sites. */
   proxyUrl?: string;
+  // ─── scrapegraph gap fields ──────────────────────────────────────────────
+  /** HTML extraction mode. 'normal'=raw, 'reader'=Readability article, 'prune'=strip nav/ads. Default: 'normal'. */
+  htmlMode?: 'normal' | 'reader' | 'prune';
+  /** Per-request screenshot dimensions and quality. Defaults: 1440×900, quality 80. */
+  screenshotConfig?: {
+    fullPage?: boolean;
+    width?: number;
+    height?: number;
+    quality?: number;
+  };
+  /** Enable/disable Playwright stealth plugin. Default: true (existing behaviour). */
+  stealth?: boolean;
+  /** Mock mode — intercept all navigation and return synthetic content. Good for CI. */
+  mock?: boolean | { minKb?: number; maxKb?: number; minSleep?: number; maxSleep?: number };
+  /** MIME type allowlist — crawler skips URLs returning disallowed content types. */
+  allowedTypes?: string[];
+  /** Glob-style include patterns for URL paths (e.g. '/blog/*'). Only matching paths crawled. */
+  includePatterns?: string[];
+  /** Glob-style exclude patterns for URL paths (e.g. '/admin/*'). Matching paths skipped. */
+  excludePatterns?: string[];
+  /** Allow crawler to follow links to external domains. Default: false. */
+  allowExternal?: boolean;
+  /** Requested output formats for a single-pass multi-format extraction. */
+  requestedFormats?: Array<'markdown' | 'html' | 'links' | 'images' | 'summary' | 'json' | 'branding' | 'screenshot'>;
+  /** LLM chunker configuration for AI extraction passes. */
+  chunkerConfig?: { size?: number | 'dynamic'; overlap?: number };
+  /** ISO-3166-1 alpha-2 country code for geo-targeted proxy routing (e.g. 'us', 'gb'). */
+  proxyCountry?: string;
+  /** When true, after the primary crawl auto-enqueue one CRAWL job per credentialProfile. */
+  crawlAllRoles?: boolean;
+  /** Decrypted credential profiles used when crawlAllRoles is true. */
+  credentialProfiles?: Array<{ id: string; name: string; username: string; password: string }>;
+  /** Set on role-crawl sub-jobs — profile name tagged on discovered screens via roleAccess column. */
+  roleProfileName?: string;
 }
 
 /** In-memory map of jobId → AbortController for RUNNING jobs. */
@@ -318,7 +352,7 @@ export async function requeueOrphanedJobs(): Promise<void> {
 
 /** Dispatches a claimed job to its type's processor. Last-resort safety net — every
  *  process*Job function already catches and records its own failures. */
-async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
+export async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
   const dispatcher = jobDispatchTable?.[job.type];
   if (!dispatcher) {
     logger.error({ jobId: job.id, type: job.type }, '[job-worker] no dispatcher registered for job type');
@@ -344,13 +378,58 @@ async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
     logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, '[job-worker] job threw uncaught error');
     await prismaAdmin.platformJob.update({
       where: { id: job.id },
-      data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() },
+      data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) },
     }).catch(() => { });
   }
 }
 
 let workerLoopTimer: ReturnType<typeof setInterval> | null = null;
 
+/** Re-promotes FAILED jobs to QUEUED if they are eligible for retry based on per-type limits. */
+export async function retryEligibleFailedJobs(): Promise<void> {
+  const rows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
+    UPDATE "PlatformJob"
+    SET status = 'QUEUED',
+        "retryCount" = "retryCount" + 1,
+        "nextRetryAt" = NOW() + (INTERVAL '1 minute' * POWER(2, "retryCount")),
+        "workerId" = NULL
+    WHERE status = 'FAILED'
+      AND dlq = false
+      AND "nextRetryAt" <= NOW()
+      AND "retryCount" < CASE
+        WHEN type IN ('CRAWL', 'NATIVE_CRAWL') THEN 3
+        WHEN type IN ('GENERATE_TESTS', 'IMPORT_TESTS', 'RUN_TEST_SUITE', 'VISUAL_REGRESSION') THEN 2
+        WHEN type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TEST_PLAN') THEN 0
+        ELSE 1
+      END
+    RETURNING id
+  `;
+  if (rows.length > 0) {
+    logger.info({ jobIds: rows.map((r: any) => r.id) }, '[job-worker] requeued failed job(s) for retry');
+  }
+}
+
+/** Promotes FAILED jobs to DLQ if they exceed their per-type retry limits. */
+export async function promoteToDeadLetterQueue(): Promise<void> {
+  const rows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
+    UPDATE "PlatformJob"
+    SET dlq = true,
+        "dlqReason" = error
+    WHERE status = 'FAILED'
+      AND dlq = false
+      AND "retryCount" >= CASE
+        WHEN type IN ('CRAWL', 'NATIVE_CRAWL') THEN 3
+        WHEN type IN ('GENERATE_TESTS', 'IMPORT_TESTS', 'RUN_TEST_SUITE', 'VISUAL_REGRESSION') THEN 2
+        WHEN type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TEST_PLAN') THEN 0
+        ELSE 1
+      END
+    RETURNING id
+  `;
+  if (rows.length > 0) {
+    logger.error({ jobIds: rows.map((r: any) => r.id) }, '[job-worker] promoted failed job(s) to DLQ');
+  }
+}
+
 /**
  * Starts the durable job worker poll loop. This is what makes job processing
  * survive a restart — enqueueJob()'s immediate same-process claim attempt is
@@ -367,9 +446,11 @@ export function startJobWorkerLoop(opts?: { pollIntervalMs?: number; concurrency
 
   const tick = async () => {
     try {
+      await retryEligibleFailedJobs();
+      await promoteToDeadLetterQueue();
       await requeueStaleRunningJobs();
     } catch (err) {
-      logger.error({ err: String(err) }, '[job-worker] stale-job sweep failed');
+      logger.error({ err: String(err) }, '[job-worker] stale/retry job sweep failed');
     }
     try {
       const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: 'queue.paused' } });
@@ -454,9 +535,9 @@ export async function enqueueJob(type: string, config: any): Promise<string> {
   // latency optimization so a job doesn't sit idle until the next poll tick. The
   // claim is atomic (WHERE status='QUEUED'), so it's safe even if the poll loop's
   // own tick fires concurrently and claims this same job first.
-  if (type === 'CRAWL' && process.env.JOB_QUEUE_BACKEND === 'sqs') {
+  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
     const { sendJobToSqs } = await import('./sqs-queue.js');
-    await sendJobToSqs(job.id, tenantId, config);
+    await sendJobToSqs(job.id);
   } else if (process.env.DISABLE_WORKER !== 'true') {
     // Skip inline dispatch when DISABLE_WORKER=true — the dedicated crawler process
     // will pick this up via its poll loop. Inline dispatch in the API container
@@ -911,8 +992,8 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
                 WHERE tc."tenantId" = ${tenantId}
                   AND s."projectId" = ${projectId}
                   AND s."elementHashChangedAt" IS NOT NULL
-                  AND tc."genAt" IS NOT NULL
-                  AND s."elementHashChangedAt" > tc."genAt"
+                  AND tc."createdAt" IS NOT NULL
+                  AND s."elementHashChangedAt" > tc."createdAt"
                   AND tc.status != 'PENDING_REVIEW'
                   AND tc.status != 'RETIRED'
               `;
@@ -967,6 +1048,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
           status: aborted ? 'CANCELLED' : 'FAILED',
           error: aborted ? 'Stopped by user' : String(err),
           finishedAt: new Date(),
+          ...(aborted ? {} : { nextRetryAt: new Date(Date.now() + 60_000) }),
           ...(checkpoint ? { result: { checkpoint } as any } : {}),
         },
       })
@@ -1051,7 +1133,7 @@ async function processApiTestsJob(jobId: string, config: any) {
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err: any) {
     await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date() } })
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) } })
     );
   }
 }
@@ -1082,7 +1164,7 @@ async function processSecurityTestsJob(jobId: string, config: any) {
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err: any) {
     await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date() } })
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) } })
     );
   }
 }
@@ -1113,7 +1195,7 @@ async function processIntegrationTestsJob(jobId: string, config: any) {
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err: any) {
     await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date() } })
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) } })
     );
   }
 }
@@ -1318,8 +1400,8 @@ async function processNativeCrawlJob(jobId: string, config: any) {
                 WHERE tc."tenantId" = ${tenantId}
                   AND s."projectId" = ${projectId}
                   AND s."elementHashChangedAt" IS NOT NULL
-                  AND tc."genAt" IS NOT NULL
-                  AND s."elementHashChangedAt" > tc."genAt"
+                  AND tc."createdAt" IS NOT NULL
+                  AND s."elementHashChangedAt" > tc."createdAt"
                   AND tc.status != 'PENDING_REVIEW'
                   AND tc.status != 'RETIRED'
               `;
@@ -1863,45 +1945,52 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
         }
         : undefined;
 
-      const authResult = await crawlProject({
-        tenantId,
-        projectId,
-        appUrl,
-        discoveryMode: true,
-        credentials: runtimeCreds,
-        storageState: crawlCfg.storageState,
-        httpBasicUsername: crawlCfg.httpBasicUsername,
-        httpBasicPassword: crawlCfg.httpBasicPassword,
-        authToken: crawlCfg.authToken,
-        authTokenHeader: crawlCfg.authTokenHeader,
-        authTokenPrefix: crawlCfg.authTokenPrefix,
-        startUrls: crawlCfg.postLoginUrl
-          ? [crawlCfg.postLoginUrl, ...(sitemapUrls.length > 0 ? sitemapUrls : [])]
-          : (sitemapUrls.length > 0 ? sitemapUrls : undefined),
-        screenshotDir,
-        maxScreens: maxDiscoverUrls,
-        proxyUrl,
-        chromeProfilePath: crawlCfg.chromeProfilePath,
-        useFirefox: crawlCfg.useFirefox,
-        storageConfig: discoverStorageConfig,
-        captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
-        captchaSolverProvider: crawlCfg.captchaSolverProvider,
-        mailslurpApiKey: crawlCfg.mailslurpApiKey,
-        mailslurpInboxId: crawlCfg.mailslurpInboxId,
-        loginInstructions: crawlCfg.loginInstructions,
-        ...resolvedSso,
-        abortSignal: controller.signal,
-        jobId,
-        onUrlsDiscovered: async (urls: string[]) => {
-          for (const u of urls) allUrls.add(u);
-          const pct = 52 + Math.round((allUrls.size / Math.max(1, maxDiscoverUrls)) * 38);
-          await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)…`);
-        },
-      }, async (pct: number, step: string) => {
-        await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
-      }).catch((err: any) => {
+      const PHASE3_TIMEOUT_MS = 20 * 60_000; // 20-minute cap on authenticated spider
+      const authResult = await Promise.race([
+        crawlProject({
+          tenantId,
+          projectId,
+          appUrl,
+          discoveryMode: true,
+          credentials: runtimeCreds,
+          storageState: crawlCfg.storageState,
+          httpBasicUsername: crawlCfg.httpBasicUsername,
+          httpBasicPassword: crawlCfg.httpBasicPassword,
+          authToken: crawlCfg.authToken,
+          authTokenHeader: crawlCfg.authTokenHeader,
+          authTokenPrefix: crawlCfg.authTokenPrefix,
+          startUrls: crawlCfg.postLoginUrl
+            ? [crawlCfg.postLoginUrl, ...(sitemapUrls.length > 0 ? sitemapUrls : [])]
+            : (sitemapUrls.length > 0 ? sitemapUrls : undefined),
+          screenshotDir,
+          maxScreens: maxDiscoverUrls,
+          proxyUrl,
+          chromeProfilePath: crawlCfg.chromeProfilePath,
+          useFirefox: crawlCfg.useFirefox,
+          storageConfig: discoverStorageConfig,
+          captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
+          captchaSolverProvider: crawlCfg.captchaSolverProvider,
+          mailslurpApiKey: crawlCfg.mailslurpApiKey,
+          mailslurpInboxId: crawlCfg.mailslurpInboxId,
+          loginInstructions: crawlCfg.loginInstructions,
+          ...resolvedSso,
+          abortSignal: controller.signal,
+          jobId,
+          onUrlsDiscovered: async (urls: string[]) => {
+            for (const u of urls) allUrls.add(u);
+            const pct = 52 + Math.round((allUrls.size / Math.max(1, maxDiscoverUrls)) * 38);
+            await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)…`);
+          },
+        }, async (pct: number, step: string) => {
+          await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
+        }),
+        new Promise<never>((_, rej) => setTimeout(() => rej(new Error('AUTH_PHASE_TIMEOUT')), PHASE3_TIMEOUT_MS)),
+      ]).catch((err: any) => {
         if (err?.message === 'CRAWL_ABORTED') throw err;
-        logger.warn({ err: err?.message }, '[discover] Phase 3 authenticated spider failed — using unauthenticated results');
+        const reason = err?.message === 'AUTH_PHASE_TIMEOUT'
+          ? 'Authenticated spider timed out after 20 minutes'
+          : err?.message;
+        logger.warn({ reason }, '[discover] Phase 3 authenticated spider failed — using unauthenticated results');
         return { discoveredUrls: [] as string[] };
       });
 
@@ -2572,6 +2661,322 @@ async function processCaptureStepsJob(jobId: string, config: any) {
   }
 }
 
+// ── Multi-role crawl orchestrator ────────────────────────────────────────────
+// Runs a primary CRAWL to discover all screens, then fans out one ROLE_CRAWL
+// job per credential profile. Role jobs run concurrently via the worker pool.
+
+async function processMultiRoleCrawlJob(jobId: string, config: any) {
+  const { tenantId, projectId } = config;
+  try {
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.updateMany({
+        where: { id: jobId, status: { not: 'CANCELLED' } },
+        data: { status: 'RUNNING', startedAt: new Date(), heartbeatAt: new Date(), workerId: WORKER_ID },
+      })
+    );
+
+    await updateJobProgress(tenantId, jobId, 5, 'Loading project crawl config…');
+
+    // Load saved project crawlConfig for primary crawl parameters
+    const proj = await withTenant(tenantId, (tx) =>
+      (tx as any).project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
+    ) as { crawlConfig: any } | null;
+    const savedCrawlConfig = proj?.crawlConfig ?? {};
+
+    const credentialProfiles: Array<{ id: string; name: string; username: string; password: string }> =
+      config.credentialProfiles ?? [];
+
+    // ── Step 1: enqueue primary CRAWL ────────────────────────────────────────
+    await updateJobProgress(tenantId, jobId, 10, 'Enqueuing primary crawl…');
+    const primaryJobId = await enqueueJob('CRAWL', {
+      tenantId,
+      projectId,
+      appUrl: config.appUrl || savedCrawlConfig.appUrl,
+      credentials: savedCrawlConfig.username && savedCrawlConfig.password
+        ? { username: savedCrawlConfig.username, password: savedCrawlConfig.password, ...(savedCrawlConfig.totpSecret ? { totpSecret: savedCrawlConfig.totpSecret } : {}) }
+        : undefined,
+      storageState: savedCrawlConfig.storageState,
+      screenshotDir: platformConfig.screenshotDir,
+      startUrls: savedCrawlConfig.seedUrls?.length ? savedCrawlConfig.seedUrls : undefined,
+      deviceProfiles: savedCrawlConfig.deviceProfiles ?? ['DESKTOP'],
+      loginInstructions: savedCrawlConfig.loginInstructions,
+      proxyUrl: savedCrawlConfig.proxyUrl,
+      useSitemap: true,
+      sitemapUrl: savedCrawlConfig.sitemapUrl,
+    });
+    logger.info({ primaryJobId, projectId }, '[multi-role-crawl] primary CRAWL enqueued');
+
+    // ── Step 2: poll until primary CRAWL is done (2-hour timeout) ────────────
+    const PRIMARY_TIMEOUT_MS = 2 * 60 * 60 * 1000;
+    const POLL_INTERVAL_MS = 10_000;
+    const deadline = Date.now() + PRIMARY_TIMEOUT_MS;
+    let primaryStatus = 'QUEUED';
+    while (Date.now() < deadline) {
+      const primaryJob = await prismaAdmin.platformJob.findUnique({
+        where: { id: primaryJobId },
+        select: { status: true },
+      }).catch(() => null);
+      primaryStatus = primaryJob?.status ?? 'UNKNOWN';
+      if (primaryStatus === 'DONE' || primaryStatus === 'FAILED' || primaryStatus === 'CANCELLED') break;
+      await updateJobProgress(tenantId, jobId, 20, `Waiting for primary crawl (${primaryStatus})…`);
+      await new Promise((res) => setTimeout(res, POLL_INTERVAL_MS));
+    }
+
+    if (primaryStatus !== 'DONE') {
+      throw new Error(`Primary crawl ${primaryJobId} ended with status ${primaryStatus} — aborting role crawls`);
+    }
+
+    // ── Step 3: collect all known screen URLs ────────────────────────────────
+    await updateJobProgress(tenantId, jobId, 50, 'Collecting screen URLs for role crawls…');
+    const screens = await prismaAdmin.screen.findMany({
+      where: { tenantId, projectId, status: 'ACTIVE' },
+      select: { id: true, url: true },
+    }).catch(() => [] as Array<{ id: string; url: string | null }>);
+    const screenUrls = screens.filter((s) => s.url).map((s) => s.url as string);
+    logger.info({ screenCount: screenUrls.length, projectId }, '[multi-role-crawl] screen URLs collected');
+
+    // ── Step 4: enqueue one ROLE_CRAWL per credential profile ────────────────
+    if (credentialProfiles.length === 0) {
+      logger.warn({ projectId, jobId }, '[multi-role-crawl] no credential profiles — nothing to role-crawl');
+    }
+
+    const roleJobs: Array<{ roleName: string; jobId: string }> = [];
+    for (const profile of credentialProfiles) {
+      if (!profile.username || !profile.password) continue;
+      const roleJobId = await enqueueJob('ROLE_CRAWL', {
+        tenantId,
+        projectId,
+        appUrl: config.appUrl || savedCrawlConfig.appUrl,
+        credentials: { username: profile.username, password: profile.password },
+        roleProfileName: profile.name,
+        parentJobId: jobId,
+        selectiveUrls: config.fullReCrawl ? undefined : (screenUrls.length > 0 ? screenUrls : undefined),
+        startUrls: config.fullReCrawl ? (savedCrawlConfig.seedUrls ?? []) : undefined,
+        maxScreens: config.fullReCrawl ? undefined : screenUrls.length + 10,
+        screenshotDir: platformConfig.screenshotDir,
+        storageState: savedCrawlConfig.storageState,
+        deviceProfiles: savedCrawlConfig.deviceProfiles ?? ['DESKTOP'],
+        loginInstructions: savedCrawlConfig.loginInstructions,
+        proxyUrl: savedCrawlConfig.proxyUrl,
+      });
+      roleJobs.push({ roleName: profile.name, jobId: roleJobId });
+      logger.info({ roleJobId, roleName: profile.name, projectId }, '[multi-role-crawl] ROLE_CRAWL enqueued');
+    }
+
+    // ── Step 5: poll until ALL role crawls complete ───────────────────────────
+    const ROLE_TIMEOUT_MS = 2 * 60 * 60 * 1000;
+    const roleDeadline = Date.now() + ROLE_TIMEOUT_MS;
+    const roleResults: Array<{ roleName: string; jobId: string; status: string; screensTagged?: number }> = [];
+
+    while (Date.now() < roleDeadline && roleJobs.length > 0) {
+      const pending: typeof roleJobs = [];
+      for (const rj of roleJobs) {
+        const rJob = await prismaAdmin.platformJob.findUnique({
+          where: { id: rj.jobId },
+          select: { status: true, result: true },
+        }).catch(() => null);
+        const st = rJob?.status ?? 'UNKNOWN';
+        if (st === 'DONE' || st === 'FAILED' || st === 'CANCELLED') {
+          roleResults.push({
+            roleName: rj.roleName,
+            jobId: rj.jobId,
+            status: st,
+            screensTagged: (rJob?.result as any)?.screensTagged,
+          });
+        } else {
+          pending.push(rj);
+        }
+      }
+      roleJobs.length = 0;
+      roleJobs.push(...pending);
+      if (roleJobs.length === 0) break;
+      const pct = Math.round(60 + ((roleResults.length / (roleResults.length + roleJobs.length)) * 35));
+      await updateJobProgress(tenantId, jobId, pct, `Role crawls: ${roleResults.length} done, ${roleJobs.length} pending…`);
+      await new Promise((res) => setTimeout(res, POLL_INTERVAL_MS));
+    }
+
+    // Any still-pending jobs timed out — record them as timed out
+    for (const rj of roleJobs) {
+      roleResults.push({ roleName: rj.roleName, jobId: rj.jobId, status: 'TIMEOUT' });
+    }
+
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({
+        where: { id: jobId },
+        data: {
+          status: 'DONE',
+          progress: 100,
+          finishedAt: new Date(),
+          result: { primaryJobId, roleJobs: roleResults } as any,
+        },
+      })
+    );
+    logger.info({ jobId, primaryJobId, roleCount: roleResults.length, projectId }, '[multi-role-crawl] completed');
+  } catch (err: any) {
+    const msg = String(err?.message ?? err);
+    logger.error({ jobId, err: msg }, '[multi-role-crawl] failed');
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: msg, finishedAt: new Date() } })
+    ).catch(() => { });
+  }
+}
+
+// ── Role-specific sub-crawl ───────────────────────────────────────────────────
+// Crawls the app as a specific credential profile and tags all discovered
+// screens with the role name in the roleAccess array column.
+
+async function processRoleCrawlJob(jobId: string, config: any) {
+  const { tenantId, projectId } = config;
+  try {
+    const claimed = await withTenant(tenantId, (tx) =>
+      tx.platformJob.updateMany({
+        where: { id: jobId, status: { not: 'CANCELLED' } },
+        data: { status: 'RUNNING', startedAt: new Date(), heartbeatAt: new Date(), workerId: WORKER_ID },
+      })
+    );
+    if (claimed.count === 0) return;
+
+    const controller = new AbortController();
+    runningControllers.set(jobId, controller);
+    const heartbeatTimer = setInterval(() => void heartbeatJob(tenantId, jobId), JOB_HEARTBEAT_INTERVAL_MS);
+
+    const onProgress = async (pct: number, step: string) => {
+      await updateJobProgress(tenantId, jobId, pct, step);
+    };
+
+    try {
+      // Load project crawlConfig for proxy, storageState overrides
+      const proj = await withTenant(tenantId, (tx) =>
+        (tx as any).project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
+      ) as { crawlConfig: any } | null;
+      const savedCrawlConfig = proj?.crawlConfig ?? {};
+
+      // Build runtime credentials with TOTP support
+      const runtimeCreds = config.credentials
+        ? {
+          ...config.credentials,
+          ...(config.credentials.totpSecret
+            ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) }
+            : {}),
+        }
+        : undefined;
+
+      const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
+
+      const result = await crawlProject({
+        ...config,
+        credentials: runtimeCreds,
+        storageState: config.storageState ?? savedCrawlConfig.storageState,
+        proxyUrl: config.proxyUrl ?? savedCrawlConfig.proxyUrl,
+        loginInstructions: config.loginInstructions ?? savedCrawlConfig.loginInstructions,
+        abortSignal: controller.signal,
+        jobId,
+        storageConfig: storageConfigForCrawl,
+      }, onProgress);
+
+      clearInterval(heartbeatTimer);
+      runningControllers.delete(jobId);
+
+      // Tag discovered screens with this role
+      const roleName = config.roleProfileName as string;
+      const discovered: any[] = (result as any).discovered ?? [];
+      const roleScreenIds = discovered.filter((d: any) => d.screenId).map((d: any) => d.screenId as string);
+
+      if (roleScreenIds.length > 0 && roleName) {
+        await prismaAdmin.$executeRaw`
+          UPDATE "Screen"
+          SET "roleAccess" = array(SELECT DISTINCT unnest(array_append("roleAccess", ${roleName}::text)))
+          WHERE id = ANY(${roleScreenIds}::text[])
+            AND "tenantId" = ${tenantId}
+        `.catch((e: any) => logger.warn({ err: e?.message, roleName, count: roleScreenIds.length }, '[role-crawl] roleAccess tag failed (non-fatal)'));
+      }
+
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            status: 'DONE',
+            progress: 100,
+            finishedAt: new Date(),
+            result: { screensTagged: roleScreenIds.length, roleName } as any,
+          },
+        })
+      );
+      logger.info({ jobId, roleName, screensTagged: roleScreenIds.length, projectId }, '[role-crawl] completed');
+    } catch (err: any) {
+      clearInterval(heartbeatTimer);
+      runningControllers.delete(jobId);
+      const aborted = String(err).includes('CRAWL_ABORTED');
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            status: aborted ? 'CANCELLED' : 'FAILED',
+            error: aborted ? 'Stopped by user' : String(err),
+            finishedAt: new Date(),
+          },
+        })
+      );
+    } finally {
+      runningControllers.delete(jobId);
+    }
+  } catch (err: any) {
+    const msg = String(err?.message ?? err);
+    logger.error({ jobId, err: msg }, '[role-crawl] outer failure');
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: msg, finishedAt: new Date() } })
+    ).catch(() => { });
+  }
+}
+
+async function processGenerateScriptJob(jobId: string, config: any): Promise<void> {
+  const { tenantId, projectId, language } = config;
+  try {
+    await updateJobProgress(tenantId, jobId, 1, 'Loading test cases…').catch(() => {});
+
+    if (config.packId) {
+      // Pack script (regression / smoke / sanity / etc.)
+      const { generatePackScript } = await import('@detiq/agents');
+      const result = await generatePackScript({ tenantId, projectId, packType: config.packId, language });
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({
+          where: { id: jobId },
+          data: { status: 'DONE', progress: 100, finishedAt: new Date(), result: result as any },
+        })
+      );
+    } else {
+      // Suite script
+      const { generateSuiteScript } = await import('@detiq/agents');
+      let progressChain: Promise<void> = Promise.resolve();
+      const result = await generateSuiteScript({
+        tenantId, projectId, suiteId: config.suiteId, language,
+        caseIds: config.caseIds ?? undefined,
+        useV2: config.useV2,
+        styleProfile: config.repoStyleProfile ?? undefined,
+        pageObjectInventory: config.repoPageObjects ?? undefined,
+        jobId,
+        onProgress: (done: number, total: number) => {
+          progressChain = progressChain
+            .then(() => updateJobProgress(tenantId, jobId, Math.floor((done / total) * 95), `Generated ${done}/${total} scripts`))
+            .catch(() => {});
+        },
+      });
+      await progressChain.catch(() => {});
+      await import('@detiq/agents')
+        .then(({ refreshScriptInventory }) => refreshScriptInventory(tenantId, projectId))
+        .catch((e: any) => logger.warn({ err: e?.message }, '[generate-script] refreshScriptInventory failed'));
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({
+          where: { id: jobId },
+          data: { status: 'DONE', progress: 100, finishedAt: new Date(), result: result as any },
+        })
+      );
+    }
+  } catch (e: any) {
+    await finishTrackedJob(tenantId, jobId, e?.message ?? 'generate-script failed');
+  }
+}
+
 // Registers the type→processor dispatch table used by the durable worker (both the
 // enqueueJob() fast-path kick and startJobWorkerLoop()'s poll). Declared at module
 // bottom so it can reference every process*Job function by name; function
@@ -2597,7 +3002,10 @@ jobDispatchTable = {
   BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
   CREATE_PR: processCreatePRJob,
   NEW_PAGES_PIPELINE: processNewPagesPipelineJob,
-  // The following job types are processed in-process by the API (never QUEUED).
+  MULTI_ROLE_CRAWL: processMultiRoleCrawlJob,
+  ROLE_CRAWL: processRoleCrawlJob,
+  GENERATE_SCRIPT: processGenerateScriptJob,
+  // The following job types are still processed in-process by the API.
   // If the worker claims one, the API crashed after creating it — mark failed so it's visible.
   GENERATE_TESTS: async (jobId: string, _config: any) => {
     logger.error({ jobId }, '[job-worker] GENERATE_TESTS claimed by worker — API did not process it in-process');
@@ -2613,13 +3021,6 @@ jobDispatchTable = {
       data: { status: 'FAILED', error: 'Data generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
     }).catch(() => { });
   },
-  GENERATE_SCRIPT: async (jobId: string, _config: any) => {
-    logger.error({ jobId }, '[job-worker] GENERATE_SCRIPT claimed by worker — API did not process it in-process');
-    await prismaAdmin.platformJob.update({
-      where: { id: jobId },
-      data: { status: 'FAILED', error: 'Script generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
-    }).catch(() => { });
-  },
   GENERATE_TEST_PLAN: async (jobId: string, _config: any) => {
     logger.error({ jobId }, '[job-worker] GENERATE_TEST_PLAN claimed by worker — API did not process it in-process');
     await prismaAdmin.platformJob.update({
@@ -2660,6 +3061,12 @@ async function processRepoScanJob(jobId: string, config: any) {
     const treeResult = await fetchRepoTree(token, conn.repoFullName, 'HEAD');
     if (treeResult.truncated) {
       console.warn(`[repo-scan] ${conn.repoFullName}: tree truncated (>100k blobs) — detection may be incomplete`);
+      await withTenant(tenantId, (tx) =>
+        tx.gitHubRepoConnection.update({
+          where: { id: repoConnectionId },
+          data: { scanError: 'Repository tree was truncated by GitHub (>100k files) — some test files in deep subdirectories may be missing from this scan.' },
+        })
+      );
     }
 
     const frameworks = detectFramework(treeResult.paths);
@@ -2687,12 +3094,20 @@ async function processRepoScanJob(jobId: string, config: any) {
         /[A-Z]\w*Tests?\.java$/.test(p) ||
         (/\/tests?\//.test(p) && /[A-Z][a-z]/.test(p.split('/').pop() ?? ''))
       );
+    // Build list of globs to try — primary + common variants so we don't miss .test.ts alongside .spec.ts
+    const testGlobs = minimatch
+      ? Array.from(new Set([
+          testGlob,
+          ...(testGlob === '**/*.spec.ts' ? ['**/*.test.ts', '**/*.cy.ts'] : []),
+          ...(testGlob === '**/*.spec.js' ? ['**/*.test.js', '**/*.cy.js'] : []),
+        ]))
+      : [];
     const testPaths = treeResult.paths.filter((p) => {
       if (language === 'java') return isJavaTestFile(p);
       if (language === 'python') return isPythonTestFile(p);
-      if (minimatch) return minimatch(p, testGlob, { matchBase: true });
-      return p.endsWith('.spec.ts') || p.endsWith('.spec.js') || p.endsWith('.test.ts') || p.endsWith('.test.js') || p.endsWith('_test.py') || p.includes('/test_');
-    }).slice(0, 500); // cap at 500 files
+      if (testGlobs.length) return testGlobs.some(g => (minimatch as any)(p, g, { matchBase: true }));
+      return p.endsWith('.spec.ts') || p.endsWith('.spec.js') || p.endsWith('.test.ts') || p.endsWith('.test.js') || p.endsWith('.cy.ts') || p.endsWith('.cy.js') || p.endsWith('_test.py') || p.includes('/test_');
+    }).slice(0, 2000); // cap at 2000 test files
 
     await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
 
@@ -3049,6 +3464,17 @@ async function processRepoScanJob(jobId: string, config: any) {
     } catch (e: any) {
       logger.warn({ jobId, repoConnectionId, err: e?.message }, '[repo-scan] repo skill files generation failed (non-fatal)');
     }
+    try {
+      const { alignSkillFile } = await import('@detiq/agents');
+      await alignSkillFile(tenantId, projectId, {
+        framework,
+        language,
+        styleProfile: resolvedStyleProfile,
+        repoFullName: conn.repoFullName,
+      });
+    } catch (e: any) {
+      logger.warn({ jobId, repoConnectionId, err: e?.message }, '[repo-scan] alignSkillFile failed (non-fatal)');
+    }
 
     await updateJobProgress(tenantId, jobId, 100, `Done: ${mapped} mapped, ${unmapped} unmapped`);
     await withTenant(tenantId, (tx) =>
@@ -3217,7 +3643,7 @@ async function processCreatePRJob(jobId: string, config: any) {
     if (testScriptIds && testScriptIds.length > 0) {
       scripts = await withTenant(tenantId, (tx) =>
         (tx as any).testScript.findMany({
-          where: { id: { in: testScriptIds }, tenantId },
+          where: { id: { in: testScriptIds }, tenantId, code: { not: null } },
           include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } },
         })
       );
@@ -3225,7 +3651,7 @@ async function processCreatePRJob(jobId: string, config: any) {
       // All scripts that haven't been pushed yet
       scripts = await withTenant(tenantId, (tx) =>
         (tx as any).testScript.findMany({
-          where: { tenantId, projectId, pushedToPR: false, validationStatus: { in: ['PASS', 'PENDING'] } },
+          where: { tenantId, projectId, pushedToPR: false, validationStatus: { in: ['PASS', 'PENDING'] }, code: { not: null } },
           include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } },
         })
       );