@detiq/crawler 1.1.28 → 1.2.0

package/src/job-queue.ts

@@ -107,6 +107,74 @@ function computeNextRetryAt(failureType: FailureType, retryCount: number): Date
   return new Date(Date.now() + delayMs);
 }
 
+/**
+ * Centralized failure recorder for all PlatformJobs.
+ * - Classifies error into FailureType (TRANSIENT, INFRASTRUCTURE, LOGICAL, FATAL)
+ * - Computes exponential backoff per failure classification
+ * - If FATAL or retries exhausted: marks status='FAILED', dlq=true, dlqAt=NOW(), dlqReason=error
+ * - If retryable: marks status='RETRY_WAIT', nextRetryAt=NOW() + backoff, retryCount=retryCount+1
+ * - Persists checkpoint and checkpointAt if checkpointData provided
+ * - Updates failureType on the row and in result.failureType
+ */
+export async function recordJobFailure(
+  tenantId: string | null | undefined,
+  jobId: string,
+  err: unknown,
+  checkpointData?: any,
+  options?: { isAborted?: boolean }
+): Promise<void> {
+  const errorMessage = err instanceof Error ? err.message : String(err);
+  const aborted = options?.isAborted ?? (errorMessage.includes('CRAWL_ABORTED') || errorMessage.includes('Stopped by user'));
+
+  if (aborted) {
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: {
+        status: 'CANCELLED',
+        error: 'Stopped by user',
+        finishedAt: new Date(),
+        heartbeatAt: null,
+        workerId: null,
+        ...(checkpointData ? { checkpoint: checkpointData, checkpointAt: new Date() } : {}),
+      },
+    }).catch(() => {});
+    return;
+  }
+
+  const failureType = classifyFailure(err);
+  const jobRow = await prismaAdmin.platformJob.findUnique({
+    where: { id: jobId },
+    select: { retryCount: true, result: true }
+  }).catch(() => null);
+
+  const currentRetry = jobRow?.retryCount ?? 0;
+  const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
+  const isFatal = !nextRetryAt || failureType === FailureType.FATAL;
+
+  await prismaAdmin.platformJob.update({
+    where: { id: jobId },
+    data: {
+      status: isFatal ? 'FAILED' : 'RETRY_WAIT',
+      error: errorMessage.slice(0, 2000),
+      finishedAt: new Date(),
+      heartbeatAt: null,
+      workerId: null,
+      nextRetryAt,
+      retryCount: { increment: 1 },
+      failureType,
+      dlq: isFatal,
+      dlqAt: isFatal ? new Date() : null,
+      dlqReason: isFatal ? errorMessage.slice(0, 1000) : null,
+      ...(checkpointData ? { checkpoint: checkpointData, checkpointAt: new Date() } : {}),
+      result: {
+        ...(jobRow?.result as any ?? {}),
+        failureType,
+        ...(checkpointData ? { checkpoint: checkpointData } : {})
+      } as any,
+    },
+  }).catch((e) => logger.error({ err: e, jobId }, '[job-queue] failed to record job failure'));
+}
+
 // ────────────────────────────────────────────────────────────────────────────────
 
 // RFC 6238 TOTP — no external dependency, uses Node built-in crypto.
@@ -362,7 +430,7 @@ export async function requeueStaleRunningJobs(): Promise<void> {
         error = 'API process was restarted while this job was running — please retry from the UI.'
     WHERE status = 'RUNNING'
       AND type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
-      AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
+      AND COALESCE("heartbeatAt", "startedAt") < (NOW() AT TIME ZONE 'UTC') - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
     RETURNING id
   `;
   if (failedRows.length > 0) {
@@ -381,7 +449,7 @@ export async function requeueStaleRunningJobs(): Promise<void> {
         )
     WHERE status = 'RUNNING'
       AND type NOT IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
-      AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
+      AND COALESCE("heartbeatAt", "startedAt") < (NOW() AT TIME ZONE 'UTC') - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
     RETURNING id
   `;
   if (rows.length > 0) {
@@ -437,39 +505,12 @@ export async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
     // rethrowing (see this function's doc comment). Read back the actual
     // final status so the metric reflects reality, not just "didn't throw".
     const finished = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { status: true } }).catch(() => null);
-    crawlJobsTotal.inc({ type: job.type, status: finished?.status === 'FAILED' ? 'failed' : 'success' });
+    crawlJobsTotal.inc({ type: job.type, status: (finished?.status === 'FAILED' || finished?.status === 'RETRY_WAIT') ? 'failed' : 'success' });
   } catch (err: any) {
     stopTimer();
     crawlJobsTotal.inc({ type: job.type, status: 'failed' });
     logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, '[job-worker] job threw uncaught error');
-    const errorMessage = String(err?.message ?? err);
-    const failureType = classifyFailure(err);
-    // Read current retryCount before incrementing so computeNextRetryAt uses the right base.
-    const jobRow = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { retryCount: true, result: true } }).catch(() => null);
-    const currentRetry = jobRow?.retryCount ?? 0;
-    const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
-    await prismaAdmin.platformJob.update({
-      where: { id: job.id },
-      data: {
-        status: nextRetryAt ? 'RETRY_WAIT' : 'FAILED',
-        error: errorMessage,
-        finishedAt: new Date(),
-        heartbeatAt: null,
-        workerId: null,
-        nextRetryAt,
-        retryCount: { increment: 1 },
-        failureType,
-        // Store failureType in result JSON for observability (backward compat)
-        result: { ...(jobRow?.result as any ?? {}), failureType } as any,
-      },
-    }).catch(() => { });
-    // If fatal or exceeded retries — promote to DLQ immediately
-    if (!nextRetryAt || failureType === FailureType.FATAL) {
-      await prismaAdmin.platformJob.update({
-        where: { id: job.id },
-        data: { dlq: true, dlqAt: new Date(), dlqReason: errorMessage },
-      }).catch(() => { });
-    }
+    await recordJobFailure(job.tenantId, job.id, err);
   }
 }
 
@@ -507,7 +548,7 @@ export async function retryEligibleFailedJobs(): Promise<void> {
     SET status = 'QUEUED',
         "requeuedAt" = NOW()
     WHERE status = 'RETRY_WAIT'
-      AND "nextRetryAt" <= NOW()
+      AND "nextRetryAt" <= (NOW() AT TIME ZONE 'UTC')
       AND dlq = false
     RETURNING id
   `;
@@ -781,10 +822,10 @@ const jobCheckpoints = new Map<string, { visitedUrls: string[]; pendingQueue: st
 export async function resumeJob(tenantId: string, jobId: string): Promise<string> {
   const job = await withTenant(tenantId, (tx) =>
     tx.platformJob.findUnique({ where: { id: jobId } })
-  ) as { type: string; config: any; result: any } | null;
+  ) as { type: string; config: any; result: any; checkpoint?: any } | null;
   if (!job) throw new Error('Job not found');
 
-  const checkpoint = (job.result as any)?.checkpoint;
+  const checkpoint = (job as any).checkpoint ?? (job.result as any)?.checkpoint;
   const visitedUrls: string[] = checkpoint?.visitedUrls ?? [];
   const pendingQueue: string[] = checkpoint?.pendingQueue ?? [];
 
@@ -796,8 +837,8 @@ export async function resumeJob(tenantId: string, jobId: string): Promise<string
   return enqueueJob(job.type, {
     ...(job.config as any),
     tenantId,
-    skipUrls: visitedUrls.length > 0 ? visitedUrls : undefined,
-    resumeUrls: pendingQueue.length > 0 ? pendingQueue : undefined,
+    skipUrls: [...((job.config as any)?.skipUrls ?? []), ...visitedUrls],
+    startUrls: pendingQueue.length > 0 ? pendingQueue : (job.config as any)?.startUrls,
   });
 }
 
@@ -1225,18 +1266,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
       }).catch((enqErr: any) => logger.warn({ err: enqErr?.message }, '[crawl] auth-failure auto-segment enqueue failed (non-fatal)'));
     }
 
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({
-        where: { id: jobId },
-        data: {
-          status: aborted ? 'CANCELLED' : 'FAILED',
-          error: aborted ? 'Stopped by user' : String(err),
-          finishedAt: new Date(),
-          ...(aborted ? {} : { nextRetryAt: new Date(Date.now() + 60_000) }),
-          ...(checkpoint ? { result: { checkpoint } as any } : {}),
-        },
-      })
-    );
+    await recordJobFailure(tenantId, jobId, err, checkpoint, { isAborted: aborted });
 
     // Notify on crawl failure (fire-and-forget)
     if (!aborted) {
@@ -1316,9 +1346,7 @@ async function processApiTestsJob(jobId: string, config: any) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err: any) {
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -1347,9 +1375,7 @@ async function processSecurityTestsJob(jobId: string, config: any) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err: any) {
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -1378,9 +1404,7 @@ async function processIntegrationTestsJob(jobId: string, config: any) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err: any) {
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -1505,9 +1529,7 @@ async function processFigmaImportJob(jobId: string, config: any) {
     });
 
   } catch (err: any) {
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -1607,9 +1629,7 @@ async function processNativeCrawlJob(jobId: string, config: any) {
       }
     });
   } catch (err: any) {
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -1622,7 +1642,7 @@ async function processGenerateTestsFromReqsJob(jobId: string, config: any) {
     const llmCfg = await getTaskAgentLLMConfig(tenantId, 'testGeneration');
 
     const all = await AppBrain.listRequirements(tenantId, projectId, {}) as any[];
-    const reqs = reqIds?.length
+    let reqs = reqIds?.length
       ? all.filter((r: any) => reqIds.includes(r.reqId))
       : all.filter((r: any) => r.status === 'unbuilt' || r.status === 'pending');
 
@@ -1646,75 +1666,107 @@ async function processGenerateTestsFromReqsJob(jobId: string, config: any) {
       resolvedSuiteId = suite.id;
     }
 
+    // Resume from checkpoint if present
+    const existingJob = await prismaAdmin.platformJob.findUnique({
+      where: { id: jobId },
+      select: { checkpoint: true },
+    }).catch(() => null);
+    const completedReqIds = new Set<string>((existingJob?.checkpoint as any)?.completedReqIds ?? []);
+    if (completedReqIds.size > 0) {
+      reqs = reqs.filter((r: any) => !completedReqIds.has(r.reqId));
+    }
+
     await updateJobProgress(tenantId, jobId, 20, `Generating tests for ${reqs.length} requirements`);
 
-    const reqBlock = reqs.map((r: any) => [
-      `## ${r.reqId}: ${r.title}`,
-      `Priority: ${r.priority} | Type: ${r.type}`,
-      r.acceptanceCriteria?.length ? `Acceptance Criteria:\n${r.acceptanceCriteria.map((c: string) => `- ${c}`).join('\n')}` : '',
-      r.businessRules?.length ? `Business Rules:\n${r.businessRules.map((b: string) => `- ${b}`).join('\n')}` : '',
-    ].filter(Boolean).join('\n')).join('\n\n');
+    const VALID_KINDS = new Set(['POSITIVE', 'NEGATIVE', 'EDGE', 'STATE_TRANSITION', 'ACCESSIBILITY', 'SECURITY', 'INTEGRATION', 'E2E', 'API', 'EXPLORATORY', 'UAT', 'PERFORMANCE', 'COMPATIBILITY', 'L10N', 'USABILITY', 'LOAD', 'UNIT', 'MUTATION', 'AB_TEST', 'ALPHA_BETA', 'SYSTEM', 'SMOKE', 'REGRESSION', 'SANITY']);
+    const VALID_PRIORITIES = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']);
 
-    const system = `You are ZeTa-AI's test case generator. Convert requirements into structured test cases.
+    // Process requirements in chunks of 5 with checkpointing
+    const CHUNK_SIZE = 5;
+    let totalCreated = (existingJob?.checkpoint as any)?.createdCount ?? 0;
+    let totalSkipped = (existingJob?.checkpoint as any)?.skippedCount ?? 0;
+    const finishedReqIds: string[] = Array.from(completedReqIds);
+
+    for (let i = 0; i < reqs.length; i += CHUNK_SIZE) {
+      const chunk = reqs.slice(i, i + CHUNK_SIZE);
+      const reqBlock = chunk.map((r: any) => [
+        `## ${r.reqId}: ${r.title}`,
+        `Priority: ${r.priority} | Type: ${r.type}`,
+        r.acceptanceCriteria?.length ? `Acceptance Criteria:\n${r.acceptanceCriteria.map((c: string) => `- ${c}`).join('\n')}` : '',
+        r.businessRules?.length ? `Business Rules:\n${r.businessRules.map((b: string) => `- ${b}`).join('\n')}` : '',
+      ].filter(Boolean).join('\n')).join('\n\n');
+
+      const system = `You are ZeTa-AI's test case generator. Convert requirements into structured test cases.
 Return ONLY valid JSON: { "cases": [ { "title": string, "kind": string, "priority": string, "preconditions": string, "steps": [{"action": string, "data": string, "expected": string}], "expectedResult": string, "tags": string[], "reqRef": string } ] }
 Kind must be one of: POSITIVE, NEGATIVE, EDGE, SECURITY, ACCESSIBILITY, INTEGRATION, E2E, SMOKE, REGRESSION.
 Priority must be one of: CRITICAL, HIGH, MEDIUM, LOW.
 Generate 1-3 test cases per requirement (positive + negative + edge if applicable).
 Each test case must reference its reqRef (the requirement ID like REQ-001).`;
 
-    const user = `Generate test cases for these requirements:\n\n${reqBlock}`;
+      const user = `Generate test cases for these requirements:\n\n${reqBlock}`;
 
-    let parsed: { cases: any[] } = { cases: [] };
-    try {
-      const reqSeed = deterministicSeed(`req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
-      const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
-      if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => { });
-      const json = text.match(/\{[\s\S]*\}/)?.[0];
-      if (json) parsed = JSON.parse(json);
-    } catch (e: any) {
-      throw new Error('AI failed to generate test cases: ' + (e.message ?? 'unknown'));
-    }
+      let parsed: { cases: any[] } = { cases: [] };
+      try {
+        const reqSeed = deterministicSeed(`req-cases:${chunk.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
+        const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
+        if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => { });
+        const json = text.match(/\{[\s\S]*\}/)?.[0];
+        if (json) parsed = JSON.parse(json);
+      } catch (e: any) {
+        throw new Error('AI failed to generate test cases: ' + (e.message ?? 'unknown'));
+      }
+
+      const normalized = (parsed.cases ?? []).map((c: any) => ({
+        ref: c.reqRef ?? null,
+        traceTo: c.reqRef ?? null,
+        title: c.title || 'Untitled',
+        kind: VALID_KINDS.has((c.kind ?? '').toUpperCase()) ? c.kind.toUpperCase() : 'POSITIVE',
+        priority: VALID_PRIORITIES.has((c.priority ?? '').toUpperCase()) ? c.priority.toUpperCase() : 'MEDIUM',
+        preconditions: c.preconditions ?? '',
+        steps: Array.isArray(c.steps) ? c.steps : [],
+        expectedResult: c.expectedResult ?? '',
+        tags: Array.isArray(c.tags) ? c.tags : [],
+      }));
 
-    await updateJobProgress(tenantId, jobId, 70, `Saving ${parsed.cases?.length ?? 0} test cases`);
+      const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
+        tenantId, projectId, resolvedSuiteId, normalized, { duplicateMode: 'skip' }
+      );
+      totalCreated += created.length;
+      totalSkipped += (skipped ?? 0);
 
-    const VALID_KINDS = new Set(['POSITIVE', 'NEGATIVE', 'EDGE', 'STATE_TRANSITION', 'ACCESSIBILITY', 'SECURITY', 'INTEGRATION', 'E2E', 'API', 'EXPLORATORY', 'UAT', 'PERFORMANCE', 'COMPATIBILITY', 'L10N', 'USABILITY', 'LOAD', 'UNIT', 'MUTATION', 'AB_TEST', 'ALPHA_BETA', 'SYSTEM', 'SMOKE', 'REGRESSION', 'SANITY']);
-    const VALID_PRIORITIES = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']);
-    const normalized = (parsed.cases ?? []).map((c: any) => ({
-      ref: c.reqRef ?? null,
-      traceTo: c.reqRef ?? null,
-      title: c.title || 'Untitled',
-      kind: VALID_KINDS.has((c.kind ?? '').toUpperCase()) ? c.kind.toUpperCase() : 'POSITIVE',
-      priority: VALID_PRIORITIES.has((c.priority ?? '').toUpperCase()) ? c.priority.toUpperCase() : 'MEDIUM',
-      preconditions: c.preconditions ?? '',
-      steps: Array.isArray(c.steps) ? c.steps : [],
-      expectedResult: c.expectedResult ?? '',
-      tags: Array.isArray(c.tags) ? c.tags : [],
-    }));
+      // Mark processed requirements as covered
+      const coveredReqIds = new Set(normalized.map((c: any) => c.ref).filter(Boolean));
+      await Promise.all(
+        chunk
+          .filter((r: any) => coveredReqIds.has(r.reqId))
+          .map((r: any) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: 'covered' }).catch(() => { }))
+      );
 
-    const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
-      tenantId, projectId, resolvedSuiteId, normalized, { duplicateMode: 'skip' }
-    );
+      for (const r of chunk) finishedReqIds.push(r.reqId);
 
-    // Mark requirements as covered
-    const coveredReqIds = new Set(normalized.map((c: any) => c.ref).filter(Boolean));
-    await Promise.all(
-      reqs
-        .filter((r: any) => coveredReqIds.has(r.reqId))
-        .map((r: any) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: 'covered' }).catch(() => { }))
-    );
+      // Save checkpoint after chunk
+      await prismaAdmin.platformJob.update({
+        where: { id: jobId },
+        data: {
+          checkpoint: { completedReqIds: finishedReqIds, createdCount: totalCreated, skippedCount: totalSkipped } as any,
+          checkpointAt: new Date(),
+        },
+      }).catch(() => {});
+
+      const progressPct = Math.round(20 + ((i + chunk.length) / reqs.length) * 75);
+      await updateJobProgress(tenantId, jobId, progressPct, `Processed ${finishedReqIds.length}/${reqs.length} requirements (${totalCreated} cases generated)`);
+    }
 
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({
         where: { id: jobId },
-        data: { status: 'DONE', finishedAt: new Date(), result: { created: created.length, skipped, suiteId: resolvedSuiteId } as any, progress: 100 },
+        data: { status: 'DONE', finishedAt: new Date(), result: { created: totalCreated, skipped: totalSkipped, suiteId: resolvedSuiteId } as any, progress: 100 },
       })
     );
   } catch (e: any) {
     const msg = e?.message ?? 'Unknown error';
     logger.error({ jobId, err: msg }, 'processGenerateTestsFromReqsJob failed');
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, e);
   }
 }
 
@@ -1768,6 +1820,21 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
     let passed = 0;
     let failed = 0;
     const runResults: Array<{ testCaseId: string; status: string }> = [];
+
+    // Resume from checkpoint if present
+    const existingJob = await prismaAdmin.platformJob.findUnique({
+      where: { id: jobId },
+      select: { checkpoint: true },
+    }).catch(() => null);
+    const completedCaseIds = new Set<string>((existingJob?.checkpoint as any)?.completedCaseIds ?? []);
+    if (completedCaseIds.size > 0) {
+      testCases = testCases.filter((tc) => !completedCaseIds.has(tc.id));
+      passed = (existingJob?.checkpoint as any)?.passed ?? 0;
+      failed = (existingJob?.checkpoint as any)?.failed ?? 0;
+    }
+
+    const allFinishedCaseIds: string[] = Array.from(completedCaseIds);
+
     for (let i = 0; i < testCases.length; i += TEST_EXEC_CONCURRENCY) {
       const batch = testCases.slice(i, i + TEST_EXEC_CONCURRENCY);
       await Promise.all(batch.map(async (tc) => {
@@ -1837,6 +1904,18 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
           }).catch(() => { });
         }
       }));
+
+      for (const tc of batch) allFinishedCaseIds.push(tc.id);
+
+      // Save durable checkpoint after each batch
+      await prismaAdmin.platformJob.update({
+        where: { id: jobId },
+        data: {
+          checkpoint: { completedCaseIds: allFinishedCaseIds, passed, failed } as any,
+          checkpointAt: new Date(),
+        },
+      }).catch(() => {});
+
       await updateJobProgress(
         tenantId, jobId,
         Math.round((Math.min(i + TEST_EXEC_CONCURRENCY, testCases.length) / testCases.length) * 100),
@@ -1874,9 +1953,7 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
     const msg = String(err?.message ?? err).slice(0, 500);
     logger.error({ jobId, testRunId, err: msg }, 'processRunTestSuiteJob failed');
     await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => { });
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -1918,9 +1995,7 @@ async function processGenerateSkillFilesJob(jobId: string, config: any) {
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', progress: 100, finishedAt: new Date() } })
     );
   } catch (err: any) {
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -2236,16 +2311,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
 
   } catch (err: any) {
     logger.error({ err: err?.message, jobId }, '[discover] Job failed');
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({
-        where: { id: jobId },
-        data: {
-          status: err?.message === 'CRAWL_ABORTED' ? 'CANCELLED' : 'FAILED',
-          result: { error: err?.message ?? String(err) } as any,
-          finishedAt: new Date(),
-        },
-      })
-    ).catch(console.error);
+    await recordJobFailure(tenantId, jobId, err);
   } finally {
     clearInterval(heartbeatTimer);
     runningControllers.delete(jobId);
@@ -2359,12 +2425,7 @@ async function processMonitorJob(jobId: string, config: any, opts?: ProcessJobOp
       })
     );
   } catch (err: any) {
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({
-        where: { id: jobId },
-        data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() },
-      })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   } finally {
     runningControllers.delete(jobId);
   }
@@ -2579,9 +2640,7 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
   } catch (err: any) {
     const msg = err?.message ?? String(err);
     logger.error({ jobId, screenId, err: msg }, 'processGenerateScreenTestsJob failed');
-    await withTenant(tenantId, (tx: any) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: msg.slice(0, 500), finishedAt: new Date() } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -2854,11 +2913,8 @@ async function processCaptureStepsJob(jobId: string, config: any) {
     );
   } catch (err: any) {
     const error = String(err?.message ?? err);
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error, finishedAt: new Date(), heartbeatAt: new Date() } })
-    ).catch(() => { });
     logger.error({ jobId, testCaseId, err: error }, 'processCaptureStepsJob failed');
-    throw err;
+    await recordJobFailure(tenantId, jobId, err);
   } finally {
     await browser?.close?.().catch(() => { });
   }
@@ -3018,9 +3074,7 @@ async function processMultiRoleCrawlJob(jobId: string, config: any) {
   } catch (err: any) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, '[multi-role-crawl] failed');
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: msg, finishedAt: new Date() } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -3126,9 +3180,7 @@ async function processRoleCrawlJob(jobId: string, config: any) {
   } catch (err: any) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, '[role-crawl] outer failure');
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: msg, finishedAt: new Date() } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -3276,9 +3328,7 @@ async function processArtifactIntelJob(jobId: string, config: any) {
     await withTenant(tenantId, (tx) =>
       (tx as any).projectArtifact.update({ where: { id: artifactId }, data: { status: 'ERROR', summary: 'AI analysis failed' } })
     ).catch(() => { });
-    await withTenant(tenantId, (tx) =>
-      (tx as any).platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
     queueImportedArtifactSkillRefresh(tenantId, projectId, 'artifact-intel-failed');
   }
 }
@@ -3789,9 +3839,7 @@ async function processRepoScanJob(jobId: string, config: any) {
     await withTenant(tenantId, (tx) =>
       tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: 'FAILED', scanError: msg.slice(0, 500) } })
     ).catch(() => { });
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -3912,9 +3960,7 @@ async function processBatchGenerateAlignedJob(jobId: string, config: any) {
   } catch (err: any) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, '[batch-generate-aligned] failed');
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -4041,9 +4087,7 @@ async function processCreatePRJob(jobId: string, config: any) {
   } catch (err: any) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, '[create-pr] failed');
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 
@@ -4145,8 +4189,6 @@ async function processNewPagesPipelineJob(jobId: string, config: any) {
   } catch (err: any) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, '[new-pages-pipeline] failed');
-    await withTenant(tenantId, (tx) =>
-      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
-    ).catch(() => { });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }

package/types/job-queue.d.ts

@@ -20,6 +20,18 @@ export declare enum FailureType {
     FATAL = "FATAL"
 }
 export declare function classifyFailure(error: string | Error | unknown): FailureType;
+/**
+ * Centralized failure recorder for all PlatformJobs.
+ * - Classifies error into FailureType (TRANSIENT, INFRASTRUCTURE, LOGICAL, FATAL)
+ * - Computes exponential backoff per failure classification
+ * - If FATAL or retries exhausted: marks status='FAILED', dlq=true, dlqAt=NOW(), dlqReason=error
+ * - If retryable: marks status='RETRY_WAIT', nextRetryAt=NOW() + backoff, retryCount=retryCount+1
+ * - Persists checkpoint and checkpointAt if checkpointData provided
+ * - Updates failureType on the row and in result.failureType
+ */
+export declare function recordJobFailure(tenantId: string | null | undefined, jobId: string, err: unknown, checkpointData?: any, options?: {
+    isAborted?: boolean;
+}): Promise<void>;
 export declare function generateTOTP(secret: string): string;
 export interface PlatformJobConfig {
     tenantId: string;