@detiq/crawler 1.1.18 → 1.1.20

package/src/job-queue.ts

@@ -240,8 +240,25 @@ export async function claimNextQueuedJob(): Promise<ClaimedJob | null> {
 
 /** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
  *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
- *  safe to run from every worker process; only affects rows past the timeout. */
+ *  safe to run from every worker process; only affects rows past the timeout.
+ *  API-in-process job types (GENERATE_SCRIPT etc.) are failed directly — requeueing them
+ *  would cause the worker to claim and fail them with a confusing "claimed by worker" error. */
 export async function requeueStaleRunningJobs(): Promise<void> {
+  // Directly fail stale API-in-process jobs — worker cannot process these types.
+  const failedRows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
+    UPDATE "PlatformJob"
+    SET status = 'FAILED',
+        "finishedAt" = NOW(),
+        error = 'API process was restarted while this job was running — please retry from the UI.'
+    WHERE status = 'RUNNING'
+      AND type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
+      AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
+    RETURNING id
+  `;
+  if (failedRows.length > 0) {
+    logger.warn({ jobIds: failedRows.map((r: any) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, '[job-worker] failed stale API-in-process job(s) — API restarted while running');
+  }
+
   const rows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
     UPDATE "PlatformJob"
     SET status = 'QUEUED',
@@ -253,6 +270,7 @@ export async function requeueStaleRunningJobs(): Promise<void> {
           true
         )
     WHERE status = 'RUNNING'
+      AND type NOT IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
       AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
     RETURNING id
   `;
@@ -691,8 +709,20 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     jobCheckpoints.delete(jobId);
 
     const pendingUrls: string[] = (result as any)?.pendingUrls ?? [];
+    let nextJobId: string | undefined;
     if (pendingUrls.length > 0) {
-      logger.info({ pendingUrls: pendingUrls.length }, 'Crawl finished with pending URLs — not auto-queuing (segmented crawls disabled)');
+      // visitedUrls includes all attempted URLs (screens, soft-404s, API paths) — more complete than
+      // discovered-with-screenId only, preventing the continuation job from re-attempting dead ends.
+      const visitedForSkip: string[] = (result as any).visitedUrls ?? ((result as any).discovered ?? [])
+        .filter((d: any) => !!d.screenId && !!d.url)
+        .map((d: any) => { try { const _u = new URL(d.url as string); _u.hash = ''; return _u.href; } catch { return d.url as string; } });
+      const reason = (result as any).wasTimeout ? 'timeout' : 'segment-limit';
+      logger.info({ pendingUrls: pendingUrls.length, visited: visitedForSkip.length, reason }, '[crawl] auto-enqueuing continuation job');
+      nextJobId = await enqueueJob('CRAWL', {
+        ...config,
+        startUrls: pendingUrls,
+        skipUrls: [...(config.skipUrls ?? []), ...visitedForSkip],
+      }).catch((autoErr: any) => { logger.warn({ err: autoErr?.message }, '[crawl] auto-segment enqueue failed (non-fatal)'); return undefined; });
     }
 
     await withTenant(tenantId, (tx) =>
@@ -700,7 +730,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
         where: { id: jobId },
         data: {
           status: 'DONE',
-          result: { ...(result as any) } as any,
+          result: { ...(result as any), ...(nextJobId ? { nextJobId } : {}) } as any,
           progress: 100,
           finishedAt: new Date(),
         },
@@ -830,7 +860,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
                 try {
                   const key = buildArtifactKey(tenantId, projectId, 'dom-snapshots', `${screen.id}.json`, tenantStorageCfg.prefix);
                   const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, domPath, key, 'application/json');
-                  await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, cloudUrl);
+                  await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, domPath, cloudUrl);
                   domUploaded++;
                 } catch (err) {
                   logger.warn({ screenId: screen.id, err }, 'Failed to upload DOM snapshot to cloud storage');
@@ -898,8 +928,23 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     clearInterval(heartbeatTimer);
     runningControllers.delete(jobId);
     const aborted = String(err).includes('CRAWL_ABORTED');
+    const authFailure = !aborted && (String(err).includes('AUTH_FAILURE') || String(err).includes('SESSION_EXPIRED'));
     const checkpoint = jobCheckpoints.get(jobId);
     jobCheckpoints.delete(jobId);
+
+    // Auth failed mid-crawl with work still pending — auto-requeue continuation job.
+    // The crawler checkpoints the remaining queue before throwing, so checkpoint.pendingQueue
+    // holds URLs not yet visited. skipUrls prevents re-crawling already-done pages.
+    if (authFailure && (checkpoint?.pendingQueue?.length ?? 0) > 0) {
+      const pendingAfterAuth: string[] = checkpoint!.pendingQueue!;
+      logger.info({ pending: pendingAfterAuth.length }, '[crawl] auth failure mid-crawl — auto-enqueuing continuation with pending URLs');
+      enqueueJob('CRAWL', {
+        ...config,
+        startUrls: pendingAfterAuth,
+        skipUrls: [...(config.skipUrls ?? []), ...(checkpoint!.visitedUrls ?? [])],
+      }).catch((enqErr: any) => logger.warn({ err: enqErr?.message }, '[crawl] auth-failure auto-segment enqueue failed (non-fatal)'));
+    }
+
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({
         where: { id: jobId },
@@ -2510,7 +2555,7 @@ jobDispatchTable = {
   BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
   CREATE_PR: processCreatePRJob,
   NEW_PAGES_PIPELINE: processNewPagesPipelineJob,
-  // GENERATE_TESTS is processed in-process by the API (never QUEUED).
+  // The following job types are processed in-process by the API (never QUEUED).
   // If the worker claims one, the API crashed after creating it — mark failed so it's visible.
   GENERATE_TESTS: async (jobId: string, _config: any) => {
     logger.error({ jobId }, '[job-worker] GENERATE_TESTS claimed by worker — API did not process it in-process');
@@ -2519,6 +2564,27 @@ jobDispatchTable = {
       data: { status: 'FAILED', error: 'Test generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
     }).catch(() => {});
   },
+  GENERATE_DATA: async (jobId: string, _config: any) => {
+    logger.error({ jobId }, '[job-worker] GENERATE_DATA claimed by worker — API did not process it in-process');
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { status: 'FAILED', error: 'Data generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
+    }).catch(() => {});
+  },
+  GENERATE_SCRIPT: async (jobId: string, _config: any) => {
+    logger.error({ jobId }, '[job-worker] GENERATE_SCRIPT claimed by worker — API did not process it in-process');
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { status: 'FAILED', error: 'Script generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
+    }).catch(() => {});
+  },
+  GENERATE_TEST_PLAN: async (jobId: string, _config: any) => {
+    logger.error({ jobId }, '[job-worker] GENERATE_TEST_PLAN claimed by worker — API did not process it in-process');
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { status: 'FAILED', error: 'Test plan generation was not completed by the API — please retry from the UI.', finishedAt: new Date() },
+    }).catch(() => {});
+  },
 };
 
 async function processRepoScanJob(jobId: string, config: any) {

package/src/native-crawler.ts

@@ -872,11 +872,12 @@ export async function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void>
         const screen = await AppBrain.saveScreen(tenantId, projectId, screenName);
         const shotPath = path.join(screenshotDir, `${screen.id}-native-v${screen.version}.png`);
         await fs.writeFile(shotPath, screenshotBuf);
-        await AppBrain.saveScreenshot(tenantId, projectId, screen.id, shotPath, screen.version, 'NATIVE');
+        let cloudUrl: string | undefined;
         if (tenantStorageCfg) {
           const s3Key = `${tenantId}/${projectId}/screenshots/${screen.id}-native-v${screen.version}.png`;
-          await uploadScreenshotToTenantStorage(tenantStorageCfg, shotPath, s3Key).catch(() => {});
+          cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, shotPath, s3Key).catch(() => undefined);
         }
+        await AppBrain.saveScreenshot(tenantId, projectId, screen.id, shotPath, screen.version, 'NATIVE', cloudUrl);
 
         // c. Parse accessibility tree → save elements
         // Scroll down to reveal below-fold elements, then scroll back up before parsing
@@ -1024,11 +1025,12 @@ export async function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void>
             const newScreen = await AppBrain.saveScreen(tenantId, projectId, newScreenName);
             const newShotPath = path.join(screenshotDir, `${newScreen.id}-native-v${newScreen.version}.png`);
             await fs.writeFile(newShotPath, Buffer.from(newScreenshot, 'base64'));
-            await AppBrain.saveScreenshot(tenantId, projectId, newScreen.id, newShotPath, newScreen.version, 'NATIVE');
+            let newCloudUrl: string | undefined;
             if (tenantStorageCfg) {
               const s3Key = `${tenantId}/${projectId}/screenshots/${newScreen.id}-native-v${newScreen.version}.png`;
-              await uploadScreenshotToTenantStorage(tenantStorageCfg, newShotPath, s3Key).catch(() => {});
+              newCloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, newShotPath, s3Key).catch(() => undefined);
             }
+            await AppBrain.saveScreenshot(tenantId, projectId, newScreen.id, newShotPath, newScreen.version, 'NATIVE', newCloudUrl);
 
             const newXml: string = await driver.getPageSource();
             const newElements = parseAccessibilityTree(newXml, platform);

package/types/crawler.d.ts

@@ -164,6 +164,8 @@ export declare function crawlProject(opts: {
     segmentIndex: number;
     hasMoreSegments: boolean;
     domHashes: string[];
+    visitedUrls?: undefined;
+    wasTimeout?: undefined;
     crawlWarnings?: undefined;
 } | {
     screensFound: number;
@@ -186,6 +188,8 @@ export declare function crawlProject(opts: {
     apiEndpoints: ApiEndpointObservationInput[];
     performanceMetrics: PerformanceMetric[];
     pendingUrls: string[];
+    visitedUrls: string[];
+    wasTimeout: boolean;
     segmentIndex: number;
     hasMoreSegments: boolean;
     domHashes: string[];

package/types/job-queue.d.ts

@@ -82,7 +82,9 @@ export declare function claimSpecificJob(jobId: string): Promise<ClaimedJob | nu
 export declare function claimNextQueuedJob(): Promise<ClaimedJob | null>;
 /** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
  *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
- *  safe to run from every worker process; only affects rows past the timeout. */
+ *  safe to run from every worker process; only affects rows past the timeout.
+ *  API-in-process job types (GENERATE_SCRIPT etc.) are failed directly — requeueing them
+ *  would cause the worker to claim and fail them with a confusing "claimed by worker" error. */
 export declare function requeueStaleRunningJobs(): Promise<void>;
 /**
  * Requeues any RUNNING jobs claimed by a different worker (i.e., a previous