@detiq/crawler 1.1.25 → 1.1.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.25",
+  "version": "1.1.26",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/job-queue.ts

@@ -67,6 +67,70 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import os from 'node:os';
 
+// ── GAP-14/15: Structured failure classification with per-type backoff ────────────────
+
+export enum FailureType {
+  TRANSIENT = 'TRANSIENT',           // network timeout, rate limit, 429, 503
+  INFRASTRUCTURE = 'INFRASTRUCTURE', // DB connection, OOM, worker crash
+  LOGICAL = 'LOGICAL',               // bad input, business rule violation, 400/422
+  FATAL = 'FATAL',                   // unrecoverable — bad config, missing secrets
+}
+
+interface BackoffConfig {
+  maxRetries: number;
+  baseDelayMs: number;
+  multiplier: number;
+  maxDelayMs: number;
+}
+
+const BACKOFF_BY_TYPE: Record<FailureType, BackoffConfig> = {
+  [FailureType.TRANSIENT]:       { maxRetries: 5, baseDelayMs: 30_000,  multiplier: 2, maxDelayMs: 300_000  },
+  [FailureType.INFRASTRUCTURE]:  { maxRetries: 3, baseDelayMs: 60_000,  multiplier: 3, maxDelayMs: 600_000  },
+  [FailureType.LOGICAL]:         { maxRetries: 1, baseDelayMs: 120_000, multiplier: 1, maxDelayMs: 120_000  },
+  [FailureType.FATAL]:           { maxRetries: 0, baseDelayMs: 0,       multiplier: 1, maxDelayMs: 0        },
+};
+
+export function classifyFailure(error: string | Error | unknown): FailureType {
+  const msg = (error instanceof Error ? error.message : String(error)).toLowerCase();
+
+  // Fatal — bad configuration, missing secrets
+  if (msg.includes('master_enc_key') || msg.includes('missing secret') || msg.includes('invalid configuration')) {
+    return FailureType.FATAL;
+  }
+
+  // Infrastructure — OOM, DB, worker crashes
+  if (
+    msg.includes('out of memory') || msg.includes('heap out') ||
+    msg.includes('connection refused') || msg.includes('econnrefused') ||
+    msg.includes('prisma') || msg.includes('database') ||
+    msg.includes('worker exited') || msg.includes('sigkill')
+  ) {
+    return FailureType.INFRASTRUCTURE;
+  }
+
+  // Logical — validation, auth, bad input
+  if (
+    msg.includes('validation') || msg.includes('invalid input') ||
+    msg.includes('not found') || msg.includes('unauthorized') ||
+    msg.includes('bad request') || msg.includes('400') || msg.includes('422') ||
+    msg.includes('403')
+  ) {
+    return FailureType.LOGICAL;
+  }
+
+  // Default: TRANSIENT (network, timeout, rate limit, 429, 503)
+  return FailureType.TRANSIENT;
+}
+
+function computeNextRetryAt(failureType: FailureType, retryCount: number): Date | null {
+  const config = BACKOFF_BY_TYPE[failureType];
+  if (retryCount >= config.maxRetries) return null; // exceeded max — go to DLQ
+  const delayMs = Math.min(config.baseDelayMs * Math.pow(config.multiplier, retryCount), config.maxDelayMs);
+  return new Date(Date.now() + delayMs);
+}
+
+// ────────────────────────────────────────────────────────────────────────────────
+
 // RFC 6238 TOTP — no external dependency, uses Node built-in crypto.
 export function generateTOTP(secret: string): string {
   const BASE32 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
@@ -227,8 +291,24 @@ const runningControllers = new Map<string, AbortController>();
 
 const WORKER_ID = `${os.hostname()}:${process.pid}`;
 const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
+const MAX_CONCURRENT_PER_TENANT = Number(process.env.MAX_CONCURRENT_PER_TENANT) || 4;
 const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15_000;
 const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
+
+// GAP-12: true active-execution semaphore — counts jobs currently executing
+// (claim→finish), not poll slots. A single poll loop checks this before every
+// claim so the cap is never exceeded even across overlapping ticks.
+let activeExecutions = 0;
+
+function withSemaphore<T>(fn: () => Promise<T>): Promise<T> {
+  activeExecutions++;
+  return fn().finally(() => { activeExecutions--; });
+}
+
+export function getActiveExecutionCount(): number {
+  return activeExecutions;
+}
+
 // Max simultaneous DISCOVER jobs — each launches a Playwright browser and holds
 // DB connections, so running many concurrently exhausts resources. Default 2.
 const MAX_CONCURRENT_DISCOVER = Number(process.env.MAX_CONCURRENT_DISCOVER) || 2;
@@ -252,6 +332,7 @@ export async function claimSpecificJob(jobId: string): Promise<ClaimedJob | null
     UPDATE "PlatformJob"
     SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
     WHERE id = ${jobId} AND status = 'QUEUED'
+      AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
     RETURNING id, "tenantId" AS "tenantId", "projectId" AS "projectId", type, config
   `;
   return rows[0] ?? null;
@@ -267,13 +348,20 @@ export async function claimNextQueuedJob(): Promise<ClaimedJob | null> {
     UPDATE "PlatformJob"
     SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
     WHERE id = (
-      SELECT id FROM "PlatformJob"
+      SELECT id FROM "PlatformJob" p
       WHERE status = 'QUEUED'
+        AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
         AND (
           type != 'DISCOVER'
           OR (SELECT COUNT(*) FROM "PlatformJob" WHERE status = 'RUNNING' AND type = 'DISCOVER') < ${maxDiscover}
         )
-      ORDER BY "queuedAt" ASC
+        AND (
+          SELECT COUNT(*) FROM "PlatformJob" p2
+          WHERE p2.status = 'RUNNING'
+            AND p2."tenantId" = p."tenantId"
+            AND p2.inline = false
+        ) < ${MAX_CONCURRENT_PER_TENANT}
+      ORDER BY p."priority" DESC, p."queuedAt" ASC
       FOR UPDATE SKIP LOCKED
       LIMIT 1
     )
@@ -376,36 +464,77 @@ export async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
     stopTimer();
     crawlJobsTotal.inc({ type: job.type, status: 'failed' });
     logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, '[job-worker] job threw uncaught error');
+    const errorMessage = String(err?.message ?? err);
+    const failureType = classifyFailure(err);
+    // Read current retryCount before incrementing so computeNextRetryAt uses the right base.
+    const jobRow = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { retryCount: true, result: true } }).catch(() => null);
+    const currentRetry = jobRow?.retryCount ?? 0;
+    const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
     await prismaAdmin.platformJob.update({
       where: { id: job.id },
-      data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) },
+      data: {
+        status: nextRetryAt ? 'RETRY_WAIT' : 'FAILED',
+        error: errorMessage,
+        finishedAt: new Date(),
+        heartbeatAt: null,
+        workerId: null,
+        nextRetryAt,
+        retryCount: { increment: 1 },
+        failureType,
+        // Store failureType in result JSON for observability (backward compat)
+        result: { ...(jobRow?.result as any ?? {}), failureType } as any,
+      },
     }).catch(() => { });
+    // If fatal or exceeded retries — promote to DLQ immediately
+    if (!nextRetryAt || failureType === FailureType.FATAL) {
+      await prismaAdmin.platformJob.update({
+        where: { id: job.id },
+        data: { dlq: true, dlqAt: new Date(), dlqReason: errorMessage },
+      }).catch(() => { });
+    }
   }
 }
 
 let workerLoopTimer: ReturnType<typeof setInterval> | null = null;
+// GAP-12: signals the semaphore-based poll loop to exit gracefully.
+let workerShuttingDown = false;
 
-/** Re-promotes FAILED jobs to QUEUED if they are eligible for retry based on per-type limits. */
+/** Re-promotes FAILED jobs to RETRY_WAIT if they are eligible for retry based on per-type limits.
+ *  RETRY_WAIT makes the waiting period visible in the UI. A separate sweep then promotes
+ *  RETRY_WAIT → QUEUED once nextRetryAt <= NOW(). */
 export async function retryEligibleFailedJobs(): Promise<void> {
+  // Legacy path: jobs that failed before failure-type classification was added.
+  // New failures go FAILED→RETRY_WAIT directly in the job error handler (dispatchClaimedJob).
+  // This only picks up old FAILED rows that have no nextRetryAt set yet.
+  // Phase 1: FAILED → RETRY_WAIT (GAP-07: visible waiting state)
   const rows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
     UPDATE "PlatformJob"
-    SET status = 'QUEUED',
+    SET status = 'RETRY_WAIT',
         "retryCount" = "retryCount" + 1,
         "nextRetryAt" = NOW() + (INTERVAL '1 minute' * POWER(2, "retryCount")),
         "workerId" = NULL
     WHERE status = 'FAILED'
       AND dlq = false
-      AND "nextRetryAt" <= NOW()
-      AND "retryCount" < CASE
-        WHEN type IN ('CRAWL', 'NATIVE_CRAWL') THEN 3
-        WHEN type IN ('GENERATE_TESTS', 'IMPORT_TESTS', 'RUN_TEST_SUITE', 'VISUAL_REGRESSION') THEN 2
-        WHEN type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TEST_PLAN') THEN 0
-        ELSE 1
-      END
+      AND "nextRetryAt" IS NULL
+      AND "retryCount" < 5
     RETURNING id
   `;
   if (rows.length > 0) {
-    logger.info({ jobIds: rows.map((r: any) => r.id) }, '[job-worker] requeued failed job(s) for retry');
+    logger.info({ jobIds: rows.map((r: any) => r.id) }, '[job-worker] moved failed job(s) to RETRY_WAIT');
+  }
+
+  // Phase 2: RETRY_WAIT → QUEUED when the wait period has elapsed (GAP-07)
+  const readyRows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
+    UPDATE "PlatformJob"
+    SET status = 'QUEUED',
+        "requeuedAt" = NOW()
+    WHERE status = 'RETRY_WAIT'
+      AND "nextRetryAt" <= NOW()
+      AND dlq = false
+    RETURNING id
+  `;
+  if (readyRows.length > 0) {
+    logger.info({ jobIds: readyRows.map((r: any) => r.id) }, '[job-worker] promoted RETRY_WAIT job(s) to QUEUED');
   }
 }
 
@@ -414,6 +543,7 @@ export async function promoteToDeadLetterQueue(): Promise<void> {
   const rows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
     UPDATE "PlatformJob"
     SET dlq = true,
+        "dlqAt" = NOW(),
         "dlqReason" = error
     WHERE status = 'FAILED'
       AND dlq = false
@@ -444,7 +574,8 @@ export function startJobWorkerLoop(opts?: { pollIntervalMs?: number; concurrency
   const concurrency = opts?.concurrency ?? WORKER_CONCURRENCY;
   logger.info({ workerId: WORKER_ID, pollIntervalMs, concurrency }, '[job-worker] starting');
 
-  const tick = async () => {
+  // GAP-12: maintenance sweep stays on a fixed interval (unchanged).
+  const sweepTick = async () => {
     try {
       await retryEligibleFailedJobs();
       await promoteToDeadLetterQueue();
@@ -452,33 +583,71 @@ export function startJobWorkerLoop(opts?: { pollIntervalMs?: number; concurrency
     } catch (err) {
       logger.error({ err: String(err) }, '[job-worker] stale/retry job sweep failed');
     }
-    try {
-      const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: 'queue.paused' } });
-      if (pauseRow?.value === 'true') {
+  };
+  workerLoopTimer = setInterval(sweepTick, pollIntervalMs);
+
+  // GAP-12: single continuous poll loop — checks the semaphore before every
+  // claim so activeExecutions never exceeds WORKER_CONCURRENCY regardless of
+  // how long individual jobs take.  Jobs are non-blocking: withSemaphore(...)
+  // is NOT awaited here, so the loop immediately loops back and can claim the
+  // next job while previous ones are still running.
+  //
+  // NOTE (GAP-20): requeueOrphanedJobs() is intentionally NOT called at startup here.
+  // Calling it at startup is unsafe in multi-replica deployments — replica A would steal
+  // RUNNING jobs from replica B (which is still alive and processing them), causing
+  // double-execution. The stale-job sweep (requeueStaleRunningJobs) handles recovery
+  // safely via heartbeat timeout. requeueOrphanedJobs() remains available for admin use.
+  workerShuttingDown = false;
+
+  const pollLoop = async () => {
+    void sweepTick(); // run one sweep immediately on startup
+    while (!workerShuttingDown) {
+      if (activeExecutions >= concurrency) {
+        // All slots busy — wait before checking again without hammering the DB.
+        await new Promise(resolve => setTimeout(resolve, 500));
+        continue;
+      }
+
+      let paused = false;
+      try {
+        const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: 'queue.paused' } });
+        paused = pauseRow?.value === 'true';
+      } catch {
+        // ignore — treat as unpaused
+      }
+
+      if (paused) {
         logger.info('[job-worker] queue is paused — skipping claim');
-      } else {
-        for (let i = 0; i < concurrency; i++) {
-          const job = await claimNextQueuedJob();
-          if (!job) break;
-          void dispatchClaimedJob(job); // not awaited — runs concurrently while the loop keeps polling
-        }
+        await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+        continue;
       }
-    } catch (err) {
-      logger.error({ err: String(err) }, '[job-worker] claim loop failed');
+
+      let job: ClaimedJob | null = null;
+      try {
+        job = await claimNextQueuedJob();
+      } catch (err) {
+        logger.error({ err: String(err) }, '[job-worker] claim failed');
+      }
+
+      if (!job) {
+        // Nothing queued — back off before polling again.
+        await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+        continue;
+      }
+
+      // Non-blocking: fire and forget into semaphore — poll loop continues immediately.
+      withSemaphore(() => dispatchClaimedJob(job!)).catch(err => {
+        logger.error({ err, jobId: job!.id }, '[worker] unhandled job error');
+      });
     }
   };
 
-  workerLoopTimer = setInterval(tick, pollIntervalMs);
-  // On startup: immediately requeue jobs orphaned by the previous process instance.
-  // This gives instant recovery rather than waiting STALE_JOB_TIMEOUT_MINUTES (30 min).
-  requeueOrphanedJobs().catch((err) =>
-    logger.error({ err: String(err) }, '[job-worker] startup orphan sweep failed')
-  );
-  void tick();
+  void pollLoop();
 }
 
 /** Stops the poll loop. Mainly for tests. */
 export function stopJobWorkerLoop(): void {
+  workerShuttingDown = true; // GAP-12: signal continuous poll loop to exit
   if (workerLoopTimer) clearInterval(workerLoopTimer);
   workerLoopTimer = null;
 }
@@ -491,31 +660,50 @@ export async function createTrackedJob(
   const name = makeJobName(type);
   const job = await withTenant(config.tenantId, (tx) =>
     tx.platformJob.create({
-      data: { tenantId: config.tenantId, projectId: config.projectId, type, name, status: 'RUNNING', startedAt: new Date(), config: config as any },
+      data: { tenantId: config.tenantId, projectId: config.projectId, type, name, status: 'RUNNING', startedAt: new Date(), inline: true, config: config as any },
     })
   ) as { id: string };
   return job.id;
 }
 
 /** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
- * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
+ * GAP-14/15: Uses failure-type classification for per-type backoff on FAILED. */
 export async function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void> {
-  const nextRetryAt = error ? new Date(Date.now() + 60_000) : undefined;
+  if (!error) {
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), progress: 100 } })
+    ).catch(() => { });
+    return;
+  }
+  const failureType = classifyFailure(error);
+  const jobRow = await prismaAdmin.platformJob.findUnique({ where: { id: jobId }, select: { retryCount: true, result: true } }).catch(() => null);
+  const currentRetry = jobRow?.retryCount ?? 0;
+  const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
       data: {
-        status: error ? 'FAILED' : 'DONE',
+        status: nextRetryAt ? 'RETRY_WAIT' : 'FAILED',
         finishedAt: new Date(),
-        ...(error ? { error, nextRetryAt } : { progress: 100 }),
+        error,
+        nextRetryAt,
+        retryCount: { increment: 1 },
+        failureType,
+        result: { ...(jobRow?.result as any ?? {}), failureType } as any,
       },
     })
   ).catch(() => { });
+  if (!nextRetryAt || failureType === FailureType.FATAL) {
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { dlq: true, dlqAt: new Date(), dlqReason: error },
+    }).catch(() => { });
+  }
 }
 
 /** Enqueue a generic platform job — returns the job ID immediately. */
 export async function enqueueJob(type: string, config: any): Promise<string> {
-  const { tenantId, projectId } = config;
+  const { tenantId, projectId, submittedBy } = config;
 
   const name = makeJobName(type);
   const job = await withTenant(tenantId, (tx) =>
@@ -527,6 +715,7 @@ export async function enqueueJob(type: string, config: any): Promise<string> {
         name,
         status: 'QUEUED',
         config: config as any,
+        ...(submittedBy ? { submittedBy } : {}),
       },
     })
   ) as { id: string };
@@ -580,7 +769,10 @@ export async function listJobs(tenantId: string, projectId: string) {
   });
 }
 
-/** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
+/** Cancel a RUNNING or QUEUED job.
+ *  Sets status to CANCEL_REQUESTED (GAP-08) so the worker's abort handler can
+ *  detect the request and set CANCELLED + finishedAt after the underlying work stops.
+ *  The in-process AbortController is also triggered for same-process jobs. */
 export async function cancelJob(tenantId: string, jobId: string): Promise<void> {
   // Abort in-memory controller if job is running in the same process
   const controller = runningControllers.get(jobId);
@@ -591,7 +783,7 @@ export async function cancelJob(tenantId: string, jobId: string): Promise<void>
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: 'CANCELLED', finishedAt: new Date(), heartbeatAt: null, workerId: null },
+      data: { status: 'CANCEL_REQUESTED' },
     })
   );
 }
@@ -636,20 +828,21 @@ export async function updateJobProgress(tenantId: string, jobId: string, progres
   try {
     await withTenant(tenantId, async (tx) => {
       // DB-flag cancellation: works for both inline and SQS backends.
-      // When a user cancels, status is set to 'CANCELLED' in DB.
+      // When a user cancels, status is set to CANCEL_REQUESTED (GAP-08).
       // This check fires at each progress update (every few seconds during crawl).
       const current = await tx.platformJob.findUnique({
         where: { id: jobId },
         select: { status: true },
       });
-      if (current?.status === 'CANCELLED') {
+      if (current?.status === 'CANCELLED' || current?.status === 'CANCEL_REQUESTED') {
         runningControllers.get(jobId)?.abort();
         runningControllers.delete(jobId);
         return; // abort signal propagates; crawlProject exits on next signal check
       }
       await tx.platformJob.update({
         where: { id: jobId },
-        data: { progress, result: { step } as any, heartbeatAt: new Date() },
+        // GAP-09: write currentStep as a proper column in addition to result.step (backward compat)
+        data: { progress, currentStep: step, result: { step } as any, heartbeatAt: new Date() },
       });
     });
   } catch { /* non-fatal — progress updates best-effort */ }
@@ -663,7 +856,7 @@ async function heartbeatJob(tenantId: string, jobId: string) {
         where: { id: jobId },
         select: { status: true },
       });
-      if (current?.status === 'CANCELLED') {
+      if (current?.status === 'CANCELLED' || current?.status === 'CANCEL_REQUESTED') {
         runningControllers.get(jobId)?.abort();
         runningControllers.delete(jobId);
         return;
@@ -800,6 +993,19 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
       storageConfig: storageConfigForCrawl,
       onCheckpoint: async (visitedUrls, pendingQueue) => {
         jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
+        // GAP-19: configurable checkpoint interval; GAP-04: persist to DB for crash recovery
+        const CHECKPOINT_INTERVAL = parseInt(process.env.CRAWL_CHECKPOINT_INTERVAL ?? '10');
+        if (visitedUrls.length % CHECKPOINT_INTERVAL === 0) {
+          try {
+            await prismaAdmin.platformJob.update({
+              where: { id: jobId },
+              data: {
+                checkpoint: { visitedUrls, pendingQueue } as any,
+                checkpointAt: new Date(),
+              },
+            });
+          } catch { /* non-fatal */ }
+        }
       },
     }, onProgress);
     jobCheckpoints.delete(jobId);
@@ -1789,6 +1995,20 @@ export async function processDataRetentionPurge(): Promise<void> {
     },
   });
   logger.info({ deleted: sessionResult.count }, '[data-retention] AuthSession purged');
+
+  // PlatformJob: delete completed jobs older than 90 days (GAP-24)
+  const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000);
+  const deletedJobs = await prismaAdmin.platformJob.deleteMany({
+    where: {
+      status: { in: ['DONE', 'FAILED'] },
+      dlq: false,
+      inline: false,
+      finishedAt: { lt: ninetyDaysAgo },
+    },
+  });
+  if (deletedJobs.count > 0) {
+    logger.info({ count: deletedJobs.count }, '[data-retention] deleted old completed jobs');
+  }
 }
 
 // ── URL Discovery Job ─────────────────────────────────────────────────────────