@detiq/crawler 0.2.13 → 1.0.0

package/dist/job-queue.js

@@ -81,11 +81,11 @@ function makeJobName(type) {
 }
 const runningControllers = /* @__PURE__ */ new Map();
 const WORKER_ID = `${os.hostname()}:${process.pid}`;
-const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES ?? 30);
-const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS ?? 15e3);
-const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY ?? 3);
-const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS ?? 15e3);
-const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY ?? 3);
+const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
+const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15e3;
+const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 3;
+const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15e3;
+const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 3;
 let jobDispatchTable = null;
 async function claimSpecificJob(jobId) {
   const rows = await prismaAdmin.$queryRaw`
@@ -1255,23 +1255,29 @@ async function processDiscoverJob(jobId, config, opts) {
   const { tenantId, projectId } = config;
   const controller = new AbortController();
   runningControllers.set(jobId, controller);
+  const heartbeatTimer = setInterval(() => {
+    void heartbeatJob(tenantId, jobId);
+  }, JOB_HEARTBEAT_INTERVAL_MS);
   const allUrls = /* @__PURE__ */ new Set();
   let lastPartialFlush = 0;
+  let lastFlushedPct = 0;
   const flushPartialUrls = async (pct, step) => {
     const now = Date.now();
     if (now - lastPartialFlush < 5e3 && allUrls.size < 5) return;
+    const monotonic = Math.max(pct, lastFlushedPct);
+    lastFlushedPct = monotonic;
     lastPartialFlush = now;
     try {
       await withTenant(
         tenantId,
         (tx) => tx.platformJob.update({
           where: { id: jobId },
-          data: { progress: pct, result: { step, partialUrls: Array.from(allUrls) }, heartbeatAt: /* @__PURE__ */ new Date() }
+          data: { progress: monotonic, result: { step, partialUrls: Array.from(allUrls) }, heartbeatAt: /* @__PURE__ */ new Date() }
         })
       );
     } catch {
     }
-    await opts?.onProgress?.(pct);
+    await opts?.onProgress?.(monotonic);
   };
   const onProgress = async (pct, step) => {
     await updateJobProgress(tenantId, jobId, pct, step);
@@ -1357,7 +1363,7 @@ async function processDiscoverJob(jobId, config, opts) {
         authToken: crawlCfg.authToken,
         authTokenHeader: crawlCfg.authTokenHeader,
         authTokenPrefix: crawlCfg.authTokenPrefix,
-        startUrls: sitemapUrls.length > 0 ? sitemapUrls : void 0,
+        startUrls: crawlCfg.postLoginUrl ? [crawlCfg.postLoginUrl, ...sitemapUrls.length > 0 ? sitemapUrls : []] : sitemapUrls.length > 0 ? sitemapUrls : void 0,
         screenshotDir,
         maxScreens: maxDiscoverUrls,
         proxyUrl,
@@ -1432,6 +1438,7 @@ async function processDiscoverJob(jobId, config, opts) {
       })
     ).catch(console.error);
   } finally {
+    clearInterval(heartbeatTimer);
     runningControllers.delete(jobId);
   }
 }