@detiq/crawler 1.1.18 → 1.1.20

package/dist/job-queue.js

@@ -113,6 +113,19 @@ async function claimNextQueuedJob() {
   return rows[0] ?? null;
 }
 async function requeueStaleRunningJobs() {
+  const failedRows = await prismaAdmin.$queryRaw`
+    UPDATE "PlatformJob"
+    SET status = 'FAILED',
+        "finishedAt" = NOW(),
+        error = 'API process was restarted while this job was running — please retry from the UI.'
+    WHERE status = 'RUNNING'
+      AND type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
+      AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
+    RETURNING id
+  `;
+  if (failedRows.length > 0) {
+    logger.warn({ jobIds: failedRows.map((r) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, "[job-worker] failed stale API-in-process job(s) \u2014 API restarted while running");
+  }
   const rows = await prismaAdmin.$queryRaw`
     UPDATE "PlatformJob"
     SET status = 'QUEUED',
@@ -124,6 +137,7 @@ async function requeueStaleRunningJobs() {
           true
         )
     WHERE status = 'RUNNING'
+      AND type NOT IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
       AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
     RETURNING id
   `;
@@ -469,8 +483,27 @@ async function processJob(jobId, config, opts) {
     }, onProgress);
     jobCheckpoints.delete(jobId);
     const pendingUrls = result?.pendingUrls ?? [];
+    let nextJobId;
     if (pendingUrls.length > 0) {
-      logger.info({ pendingUrls: pendingUrls.length }, "Crawl finished with pending URLs \u2014 not auto-queuing (segmented crawls disabled)");
+      const visitedForSkip = result.visitedUrls ?? (result.discovered ?? []).filter((d) => !!d.screenId && !!d.url).map((d) => {
+        try {
+          const _u = new URL(d.url);
+          _u.hash = "";
+          return _u.href;
+        } catch {
+          return d.url;
+        }
+      });
+      const reason = result.wasTimeout ? "timeout" : "segment-limit";
+      logger.info({ pendingUrls: pendingUrls.length, visited: visitedForSkip.length, reason }, "[crawl] auto-enqueuing continuation job");
+      nextJobId = await enqueueJob("CRAWL", {
+        ...config,
+        startUrls: pendingUrls,
+        skipUrls: [...config.skipUrls ?? [], ...visitedForSkip]
+      }).catch((autoErr) => {
+        logger.warn({ err: autoErr?.message }, "[crawl] auto-segment enqueue failed (non-fatal)");
+        return void 0;
+      });
     }
     await withTenant(
       tenantId,
@@ -478,7 +511,7 @@ async function processJob(jobId, config, opts) {
         where: { id: jobId },
         data: {
           status: "DONE",
-          result: { ...result },
+          result: { ...result, ...nextJobId ? { nextJobId } : {} },
           progress: 100,
           finishedAt: /* @__PURE__ */ new Date()
         }
@@ -586,7 +619,7 @@ async function processJob(jobId, config, opts) {
                 try {
                   const key = buildArtifactKey(tenantId, projectId, "dom-snapshots", `${screen.id}.json`, tenantStorageCfg.prefix);
                   const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, domPath, key, "application/json");
-                  await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, cloudUrl);
+                  await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, domPath, cloudUrl);
                   domUploaded++;
                 } catch (err) {
                   logger.warn({ screenId: screen.id, err }, "Failed to upload DOM snapshot to cloud storage");
@@ -647,8 +680,18 @@ async function processJob(jobId, config, opts) {
     clearInterval(heartbeatTimer);
     runningControllers.delete(jobId);
     const aborted = String(err).includes("CRAWL_ABORTED");
+    const authFailure = !aborted && (String(err).includes("AUTH_FAILURE") || String(err).includes("SESSION_EXPIRED"));
     const checkpoint = jobCheckpoints.get(jobId);
     jobCheckpoints.delete(jobId);
+    if (authFailure && (checkpoint?.pendingQueue?.length ?? 0) > 0) {
+      const pendingAfterAuth = checkpoint.pendingQueue;
+      logger.info({ pending: pendingAfterAuth.length }, "[crawl] auth failure mid-crawl \u2014 auto-enqueuing continuation with pending URLs");
+      enqueueJob("CRAWL", {
+        ...config,
+        startUrls: pendingAfterAuth,
+        skipUrls: [...config.skipUrls ?? [], ...checkpoint.visitedUrls ?? []]
+      }).catch((enqErr) => logger.warn({ err: enqErr?.message }, "[crawl] auth-failure auto-segment enqueue failed (non-fatal)"));
+    }
     await withTenant(
       tenantId,
       (tx) => tx.platformJob.update({
@@ -2076,7 +2119,7 @@ jobDispatchTable = {
   BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
   CREATE_PR: processCreatePRJob,
   NEW_PAGES_PIPELINE: processNewPagesPipelineJob,
-  // GENERATE_TESTS is processed in-process by the API (never QUEUED).
+  // The following job types are processed in-process by the API (never QUEUED).
   // If the worker claims one, the API crashed after creating it — mark failed so it's visible.
   GENERATE_TESTS: async (jobId, _config) => {
     logger.error({ jobId }, "[job-worker] GENERATE_TESTS claimed by worker \u2014 API did not process it in-process");
@@ -2085,6 +2128,30 @@ jobDispatchTable = {
       data: { status: "FAILED", error: "Test generation was not completed by the API \u2014 please retry from the UI.", finishedAt: /* @__PURE__ */ new Date() }
     }).catch(() => {
     });
+  },
+  GENERATE_DATA: async (jobId, _config) => {
+    logger.error({ jobId }, "[job-worker] GENERATE_DATA claimed by worker \u2014 API did not process it in-process");
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { status: "FAILED", error: "Data generation was not completed by the API \u2014 please retry from the UI.", finishedAt: /* @__PURE__ */ new Date() }
+    }).catch(() => {
+    });
+  },
+  GENERATE_SCRIPT: async (jobId, _config) => {
+    logger.error({ jobId }, "[job-worker] GENERATE_SCRIPT claimed by worker \u2014 API did not process it in-process");
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { status: "FAILED", error: "Script generation was not completed by the API \u2014 please retry from the UI.", finishedAt: /* @__PURE__ */ new Date() }
+    }).catch(() => {
+    });
+  },
+  GENERATE_TEST_PLAN: async (jobId, _config) => {
+    logger.error({ jobId }, "[job-worker] GENERATE_TEST_PLAN claimed by worker \u2014 API did not process it in-process");
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { status: "FAILED", error: "Test plan generation was not completed by the API \u2014 please retry from the UI.", finishedAt: /* @__PURE__ */ new Date() }
+    }).catch(() => {
+    });
   }
 };
 async function processRepoScanJob(jobId, config) {