@detiq/crawler 1.1.19 → 1.1.20

package/src/job-queue.ts

@@ -709,8 +709,20 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     jobCheckpoints.delete(jobId);
 
     const pendingUrls: string[] = (result as any)?.pendingUrls ?? [];
+    let nextJobId: string | undefined;
     if (pendingUrls.length > 0) {
-      logger.info({ pendingUrls: pendingUrls.length }, 'Crawl finished with pending URLs — not auto-queuing (segmented crawls disabled)');
+      // visitedUrls includes all attempted URLs (screens, soft-404s, API paths) — more complete than
+      // discovered-with-screenId only, preventing the continuation job from re-attempting dead ends.
+      const visitedForSkip: string[] = (result as any).visitedUrls ?? ((result as any).discovered ?? [])
+        .filter((d: any) => !!d.screenId && !!d.url)
+        .map((d: any) => { try { const _u = new URL(d.url as string); _u.hash = ''; return _u.href; } catch { return d.url as string; } });
+      const reason = (result as any).wasTimeout ? 'timeout' : 'segment-limit';
+      logger.info({ pendingUrls: pendingUrls.length, visited: visitedForSkip.length, reason }, '[crawl] auto-enqueuing continuation job');
+      nextJobId = await enqueueJob('CRAWL', {
+        ...config,
+        startUrls: pendingUrls,
+        skipUrls: [...(config.skipUrls ?? []), ...visitedForSkip],
+      }).catch((autoErr: any) => { logger.warn({ err: autoErr?.message }, '[crawl] auto-segment enqueue failed (non-fatal)'); return undefined; });
     }
 
     await withTenant(tenantId, (tx) =>
@@ -718,7 +730,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
         where: { id: jobId },
         data: {
           status: 'DONE',
-          result: { ...(result as any) } as any,
+          result: { ...(result as any), ...(nextJobId ? { nextJobId } : {}) } as any,
           progress: 100,
           finishedAt: new Date(),
         },
@@ -848,7 +860,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
                 try {
                   const key = buildArtifactKey(tenantId, projectId, 'dom-snapshots', `${screen.id}.json`, tenantStorageCfg.prefix);
                   const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, domPath, key, 'application/json');
-                  await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, cloudUrl);
+                  await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, domPath, cloudUrl);
                   domUploaded++;
                 } catch (err) {
                   logger.warn({ screenId: screen.id, err }, 'Failed to upload DOM snapshot to cloud storage');
@@ -916,8 +928,23 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     clearInterval(heartbeatTimer);
     runningControllers.delete(jobId);
     const aborted = String(err).includes('CRAWL_ABORTED');
+    const authFailure = !aborted && (String(err).includes('AUTH_FAILURE') || String(err).includes('SESSION_EXPIRED'));
     const checkpoint = jobCheckpoints.get(jobId);
     jobCheckpoints.delete(jobId);
+
+    // Auth failed mid-crawl with work still pending — auto-requeue continuation job.
+    // The crawler checkpoints the remaining queue before throwing, so checkpoint.pendingQueue
+    // holds URLs not yet visited. skipUrls prevents re-crawling already-done pages.
+    if (authFailure && (checkpoint?.pendingQueue?.length ?? 0) > 0) {
+      const pendingAfterAuth: string[] = checkpoint!.pendingQueue!;
+      logger.info({ pending: pendingAfterAuth.length }, '[crawl] auth failure mid-crawl — auto-enqueuing continuation with pending URLs');
+      enqueueJob('CRAWL', {
+        ...config,
+        startUrls: pendingAfterAuth,
+        skipUrls: [...(config.skipUrls ?? []), ...(checkpoint!.visitedUrls ?? [])],
+      }).catch((enqErr: any) => logger.warn({ err: enqErr?.message }, '[crawl] auth-failure auto-segment enqueue failed (non-fatal)'));
+    }
+
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({
         where: { id: jobId },

package/src/native-crawler.ts

@@ -872,11 +872,12 @@ export async function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void>
         const screen = await AppBrain.saveScreen(tenantId, projectId, screenName);
         const shotPath = path.join(screenshotDir, `${screen.id}-native-v${screen.version}.png`);
         await fs.writeFile(shotPath, screenshotBuf);
-        await AppBrain.saveScreenshot(tenantId, projectId, screen.id, shotPath, screen.version, 'NATIVE');
+        let cloudUrl: string | undefined;
         if (tenantStorageCfg) {
           const s3Key = `${tenantId}/${projectId}/screenshots/${screen.id}-native-v${screen.version}.png`;
-          await uploadScreenshotToTenantStorage(tenantStorageCfg, shotPath, s3Key).catch(() => {});
+          cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, shotPath, s3Key).catch(() => undefined);
         }
+        await AppBrain.saveScreenshot(tenantId, projectId, screen.id, shotPath, screen.version, 'NATIVE', cloudUrl);
 
         // c. Parse accessibility tree → save elements
         // Scroll down to reveal below-fold elements, then scroll back up before parsing
@@ -1024,11 +1025,12 @@ export async function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void>
             const newScreen = await AppBrain.saveScreen(tenantId, projectId, newScreenName);
             const newShotPath = path.join(screenshotDir, `${newScreen.id}-native-v${newScreen.version}.png`);
             await fs.writeFile(newShotPath, Buffer.from(newScreenshot, 'base64'));
-            await AppBrain.saveScreenshot(tenantId, projectId, newScreen.id, newShotPath, newScreen.version, 'NATIVE');
+            let newCloudUrl: string | undefined;
             if (tenantStorageCfg) {
               const s3Key = `${tenantId}/${projectId}/screenshots/${newScreen.id}-native-v${newScreen.version}.png`;
-              await uploadScreenshotToTenantStorage(tenantStorageCfg, newShotPath, s3Key).catch(() => {});
+              newCloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, newShotPath, s3Key).catch(() => undefined);
             }
+            await AppBrain.saveScreenshot(tenantId, projectId, newScreen.id, newShotPath, newScreen.version, 'NATIVE', newCloudUrl);
 
             const newXml: string = await driver.getPageSource();
             const newElements = parseAccessibilityTree(newXml, platform);

package/types/crawler.d.ts

@@ -164,6 +164,8 @@ export declare function crawlProject(opts: {
     segmentIndex: number;
     hasMoreSegments: boolean;
     domHashes: string[];
+    visitedUrls?: undefined;
+    wasTimeout?: undefined;
     crawlWarnings?: undefined;
 } | {
     screensFound: number;
@@ -186,6 +188,8 @@ export declare function crawlProject(opts: {
     apiEndpoints: ApiEndpointObservationInput[];
     performanceMetrics: PerformanceMetric[];
     pendingUrls: string[];
+    visitedUrls: string[];
+    wasTimeout: boolean;
     segmentIndex: number;
     hasMoreSegments: boolean;
     domHashes: string[];

package/types/job-queue.d.ts

@@ -82,7 +82,9 @@ export declare function claimSpecificJob(jobId: string): Promise<ClaimedJob | nu
 export declare function claimNextQueuedJob(): Promise<ClaimedJob | null>;
 /** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
  *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
- *  safe to run from every worker process; only affects rows past the timeout. */
+ *  safe to run from every worker process; only affects rows past the timeout.
+ *  API-in-process job types (GENERATE_SCRIPT etc.) are failed directly — requeueing them
+ *  would cause the worker to claim and fail them with a confusing "claimed by worker" error. */
 export declare function requeueStaleRunningJobs(): Promise<void>;
 /**
  * Requeues any RUNNING jobs claimed by a different worker (i.e., a previous