@detiq/crawler 1.1.28 → 1.2.0

package/dist/failure-analysis-agent.js

@@ -23,7 +23,8 @@ Error: ${ctx.error}${consoleSection}${networkSection}`;
     if (ctx.screenshotPath) {
       try {
         const img = readFileSync(ctx.screenshotPath);
-        const result2 = await askTrackedWithImages("", userText, [{ base64: img.toString("base64"), mediaType: "image/png" }], llmCfg);
+        const imgSeed = deterministicSeed(`failure-img:${ctx.tenantId ?? ""}:${(ctx.stepDescription ?? "").slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+        const result2 = await askTrackedWithImages("", userText, [{ base64: img.toString("base64"), mediaType: "image/png" }], llmCfg, { temperature: 0, seed: imgSeed });
         return result2.text || null;
       } catch {
       }

package/dist/failure-analysis-agent.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/failure-analysis-agent.ts"],
-  "sourcesContent": ["import { readFileSync } from 'node:fs';\nimport { askTrackedWithImages, askTracked, deterministicSeed } from '@detiq/agents';\nimport type { LLMConfig } from '@detiq/agents';\nimport { AppBrain } from '@detiq/app-brain';\n\nexport interface FailureContext {\n  tenantId?: string;\n  error: string;\n  stepDescription?: string;\n  consoleLogs?: Array<{ type: string; text: string }>;\n  networkErrors?: Array<{ url: string; status: number; method: string }>;\n  screenshotPath?: string;\n}\n\nexport async function analyzeFailure(ctx: FailureContext): Promise<string | null> {\n  try {\n    let llmCfg: LLMConfig | undefined;\n    if (ctx.tenantId) {\n      llmCfg = await AppBrain.getTaskLLMConfig(ctx.tenantId, 'testRunner').catch(() => undefined) as LLMConfig | undefined ?? undefined;\n    }\n    if (!llmCfg) return null;\n\n    const consoleSection = ctx.consoleLogs?.length\n      ? `\\n\\nBrowser console (last ${ctx.consoleLogs.length} entries):\\n${ctx.consoleLogs.map(l => `[${l.type}] ${l.text}`).join('\\n')}`\n      : '';\n\n    const networkSection = ctx.networkErrors?.length\n      ? `\\n\\nFailed network requests:\\n${ctx.networkErrors.map(n => `${n.method} ${n.url} \u2192 ${n.status}`).join('\\n')}`\n      : '';\n\n    const userText = `A Playwright automated test failed. Identify the root cause in 2-3 sentences. Be specific \u2014 name the failing element, API endpoint, or error type. Do not suggest fixes.\n\nTest step: ${ctx.stepDescription ?? 'unknown'}\nError: ${ctx.error}${consoleSection}${networkSection}`;\n\n    if (ctx.screenshotPath) {\n      try {\n        const img = readFileSync(ctx.screenshotPath);\n        const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg);\n        return result.text || null;\n      } catch { /* fall through to text-only */ }\n    }\n\n    const failureSeed = deterministicSeed(`${ctx.tenantId ?? ''}:failure:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);\n    const result = await askTracked('', userText, llmCfg, 256, undefined, { temperature: 0, seed: failureSeed });\n    return result.text || null;\n  } catch {\n    return null;\n  }\n}\n"],
-  "mappings": "AAAA,SAAS,oBAAoB;AAC7B,SAAS,sBAAsB,YAAY,yBAAyB;AAEpE,SAAS,gBAAgB;AAWzB,eAAsB,eAAe,KAA6C;AAChF,MAAI;AACF,QAAI;AACJ,QAAI,IAAI,UAAU;AAChB,eAAS,MAAM,SAAS,iBAAiB,IAAI,UAAU,YAAY,EAAE,MAAM,MAAM,MAAS,KAA8B;AAAA,IAC1H;AACA,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,iBAAiB,IAAI,aAAa,SACpC;AAAA;AAAA,wBAA6B,IAAI,YAAY,MAAM;AAAA,EAAe,IAAI,YAAY,IAAI,OAAK,IAAI,EAAE,IAAI,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,IAAI,CAAC,KAC9H;AAEJ,UAAM,iBAAiB,IAAI,eAAe,SACtC;AAAA;AAAA;AAAA,EAAiC,IAAI,cAAc,IAAI,OAAK,GAAG,EAAE,MAAM,IAAI,EAAE,GAAG,WAAM,EAAE,MAAM,EAAE,EAAE,KAAK,IAAI,CAAC,KAC5G;AAEJ,UAAM,WAAW;AAAA;AAAA,aAER,IAAI,mBAAmB,SAAS;AAAA,SACpC,IAAI,KAAK,GAAG,cAAc,GAAG,cAAc;AAEhD,QAAI,IAAI,gBAAgB;AACtB,UAAI;AACF,cAAM,MAAM,aAAa,IAAI,cAAc;AAC3C,cAAMA,UAAS,MAAM,qBAAqB,IAAI,UAAU,CAAC,EAAE,QAAQ,IAAI,SAAS,QAAQ,GAAG,WAAW,YAAY,CAAC,GAAG,MAAM;AAC5H,eAAOA,QAAO,QAAQ;AAAA,MACxB,QAAQ;AAAA,MAAkC;AAAA,IAC5C;AAEA,UAAM,cAAc,kBAAkB,GAAG,IAAI,YAAY,EAAE,aAAa,IAAI,mBAAmB,IAAI,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE;AAC7I,UAAM,SAAS,MAAM,WAAW,IAAI,UAAU,QAAQ,KAAK,QAAW,EAAE,aAAa,GAAG,MAAM,YAAY,CAAC;AAC3G,WAAO,OAAO,QAAQ;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "sourcesContent": ["import { readFileSync } from 'node:fs';\nimport { askTrackedWithImages, askTracked, deterministicSeed } from '@detiq/agents';\nimport type { LLMConfig } from '@detiq/agents';\nimport { AppBrain } from '@detiq/app-brain';\n\nexport interface FailureContext {\n  tenantId?: string;\n  error: string;\n  stepDescription?: string;\n  consoleLogs?: Array<{ type: string; text: string }>;\n  networkErrors?: Array<{ url: string; status: number; method: string }>;\n  screenshotPath?: string;\n}\n\nexport async function analyzeFailure(ctx: FailureContext): Promise<string | null> {\n  try {\n    let llmCfg: LLMConfig | undefined;\n    if (ctx.tenantId) {\n      llmCfg = await AppBrain.getTaskLLMConfig(ctx.tenantId, 'testRunner').catch(() => undefined) as LLMConfig | undefined ?? undefined;\n    }\n    if (!llmCfg) return null;\n\n    const consoleSection = ctx.consoleLogs?.length\n      ? `\\n\\nBrowser console (last ${ctx.consoleLogs.length} entries):\\n${ctx.consoleLogs.map(l => `[${l.type}] ${l.text}`).join('\\n')}`\n      : '';\n\n    const networkSection = ctx.networkErrors?.length\n      ? `\\n\\nFailed network requests:\\n${ctx.networkErrors.map(n => `${n.method} ${n.url} \u2192 ${n.status}`).join('\\n')}`\n      : '';\n\n    const userText = `A Playwright automated test failed. Identify the root cause in 2-3 sentences. Be specific \u2014 name the failing element, API endpoint, or error type. Do not suggest fixes.\n\nTest step: ${ctx.stepDescription ?? 'unknown'}\nError: ${ctx.error}${consoleSection}${networkSection}`;\n\n    if (ctx.screenshotPath) {\n      try {\n        const img = readFileSync(ctx.screenshotPath);\n        const imgSeed = deterministicSeed(`failure-img:${ctx.tenantId ?? ''}:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);\n        const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg, { temperature: 0, seed: imgSeed });\n        return result.text || null;\n      } catch { /* fall through to text-only */ }\n    }\n\n    const failureSeed = deterministicSeed(`${ctx.tenantId ?? ''}:failure:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);\n    const result = await askTracked('', userText, llmCfg, 256, undefined, { temperature: 0, seed: failureSeed });\n    return result.text || null;\n  } catch {\n    return null;\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,oBAAoB;AAC7B,SAAS,sBAAsB,YAAY,yBAAyB;AAEpE,SAAS,gBAAgB;AAWzB,eAAsB,eAAe,KAA6C;AAChF,MAAI;AACF,QAAI;AACJ,QAAI,IAAI,UAAU;AAChB,eAAS,MAAM,SAAS,iBAAiB,IAAI,UAAU,YAAY,EAAE,MAAM,MAAM,MAAS,KAA8B;AAAA,IAC1H;AACA,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,iBAAiB,IAAI,aAAa,SACpC;AAAA;AAAA,wBAA6B,IAAI,YAAY,MAAM;AAAA,EAAe,IAAI,YAAY,IAAI,OAAK,IAAI,EAAE,IAAI,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,IAAI,CAAC,KAC9H;AAEJ,UAAM,iBAAiB,IAAI,eAAe,SACtC;AAAA;AAAA;AAAA,EAAiC,IAAI,cAAc,IAAI,OAAK,GAAG,EAAE,MAAM,IAAI,EAAE,GAAG,WAAM,EAAE,MAAM,EAAE,EAAE,KAAK,IAAI,CAAC,KAC5G;AAEJ,UAAM,WAAW;AAAA;AAAA,aAER,IAAI,mBAAmB,SAAS;AAAA,SACpC,IAAI,KAAK,GAAG,cAAc,GAAG,cAAc;AAEhD,QAAI,IAAI,gBAAgB;AACtB,UAAI;AACF,cAAM,MAAM,aAAa,IAAI,cAAc;AAC3C,cAAM,UAAU,kBAAkB,eAAe,IAAI,YAAY,EAAE,KAAK,IAAI,mBAAmB,IAAI,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE;AAC7I,cAAMA,UAAS,MAAM,qBAAqB,IAAI,UAAU,CAAC,EAAE,QAAQ,IAAI,SAAS,QAAQ,GAAG,WAAW,YAAY,CAAC,GAAG,QAAQ,EAAE,aAAa,GAAG,MAAM,QAAQ,CAAC;AAC/J,eAAOA,QAAO,QAAQ;AAAA,MACxB,QAAQ;AAAA,MAAkC;AAAA,IAC5C;AAEA,UAAM,cAAc,kBAAkB,GAAG,IAAI,YAAY,EAAE,aAAa,IAAI,mBAAmB,IAAI,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE;AAC7I,UAAM,SAAS,MAAM,WAAW,IAAI,UAAU,QAAQ,KAAK,QAAW,EAAE,aAAa,GAAG,MAAM,YAAY,CAAC;AAC3G,WAAO,OAAO,QAAQ;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
   "names": ["result"]
 }

package/dist/job-queue.js

@@ -55,6 +55,55 @@ function computeNextRetryAt(failureType, retryCount) {
   const delayMs = Math.min(config.baseDelayMs * Math.pow(config.multiplier, retryCount), config.maxDelayMs);
   return new Date(Date.now() + delayMs);
 }
+async function recordJobFailure(tenantId, jobId, err, checkpointData, options) {
+  const errorMessage = err instanceof Error ? err.message : String(err);
+  const aborted = options?.isAborted ?? (errorMessage.includes("CRAWL_ABORTED") || errorMessage.includes("Stopped by user"));
+  if (aborted) {
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: {
+        status: "CANCELLED",
+        error: "Stopped by user",
+        finishedAt: /* @__PURE__ */ new Date(),
+        heartbeatAt: null,
+        workerId: null,
+        ...checkpointData ? { checkpoint: checkpointData, checkpointAt: /* @__PURE__ */ new Date() } : {}
+      }
+    }).catch(() => {
+    });
+    return;
+  }
+  const failureType = classifyFailure(err);
+  const jobRow = await prismaAdmin.platformJob.findUnique({
+    where: { id: jobId },
+    select: { retryCount: true, result: true }
+  }).catch(() => null);
+  const currentRetry = jobRow?.retryCount ?? 0;
+  const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
+  const isFatal = !nextRetryAt || failureType === "FATAL" /* FATAL */;
+  await prismaAdmin.platformJob.update({
+    where: { id: jobId },
+    data: {
+      status: isFatal ? "FAILED" : "RETRY_WAIT",
+      error: errorMessage.slice(0, 2e3),
+      finishedAt: /* @__PURE__ */ new Date(),
+      heartbeatAt: null,
+      workerId: null,
+      nextRetryAt,
+      retryCount: { increment: 1 },
+      failureType,
+      dlq: isFatal,
+      dlqAt: isFatal ? /* @__PURE__ */ new Date() : null,
+      dlqReason: isFatal ? errorMessage.slice(0, 1e3) : null,
+      ...checkpointData ? { checkpoint: checkpointData, checkpointAt: /* @__PURE__ */ new Date() } : {},
+      result: {
+        ...jobRow?.result ?? {},
+        failureType,
+        ...checkpointData ? { checkpoint: checkpointData } : {}
+      }
+    }
+  }).catch((e) => logger.error({ err: e, jobId }, "[job-queue] failed to record job failure"));
+}
 function generateTOTP(secret) {
   const BASE32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
   const clean = secret.toUpperCase().replace(/\s/g, "").replace(/=/g, "");
@@ -156,7 +205,7 @@ async function requeueStaleRunningJobs() {
         error = 'API process was restarted while this job was running — please retry from the UI.'
     WHERE status = 'RUNNING'
       AND type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
-      AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
+      AND COALESCE("heartbeatAt", "startedAt") < (NOW() AT TIME ZONE 'UTC') - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
     RETURNING id
   `;
   if (failedRows.length > 0) {
@@ -174,7 +223,7 @@ async function requeueStaleRunningJobs() {
         )
     WHERE status = 'RUNNING'
       AND type NOT IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TESTS', 'GENERATE_TEST_PLAN')
-      AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
+      AND COALESCE("heartbeatAt", "startedAt") < (NOW() AT TIME ZONE 'UTC') - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
     RETURNING id
   `;
   if (rows.length > 0) {
@@ -217,39 +266,12 @@ async function dispatchClaimedJob(job) {
     await dispatcher(job.id, job.config);
     stopTimer();
     const finished = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { status: true } }).catch(() => null);
-    crawlJobsTotal.inc({ type: job.type, status: finished?.status === "FAILED" ? "failed" : "success" });
+    crawlJobsTotal.inc({ type: job.type, status: finished?.status === "FAILED" || finished?.status === "RETRY_WAIT" ? "failed" : "success" });
   } catch (err) {
     stopTimer();
     crawlJobsTotal.inc({ type: job.type, status: "failed" });
     logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, "[job-worker] job threw uncaught error");
-    const errorMessage = String(err?.message ?? err);
-    const failureType = classifyFailure(err);
-    const jobRow = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { retryCount: true, result: true } }).catch(() => null);
-    const currentRetry = jobRow?.retryCount ?? 0;
-    const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
-    await prismaAdmin.platformJob.update({
-      where: { id: job.id },
-      data: {
-        status: nextRetryAt ? "RETRY_WAIT" : "FAILED",
-        error: errorMessage,
-        finishedAt: /* @__PURE__ */ new Date(),
-        heartbeatAt: null,
-        workerId: null,
-        nextRetryAt,
-        retryCount: { increment: 1 },
-        failureType,
-        // Store failureType in result JSON for observability (backward compat)
-        result: { ...jobRow?.result ?? {}, failureType }
-      }
-    }).catch(() => {
-    });
-    if (!nextRetryAt || failureType === "FATAL" /* FATAL */) {
-      await prismaAdmin.platformJob.update({
-        where: { id: job.id },
-        data: { dlq: true, dlqAt: /* @__PURE__ */ new Date(), dlqReason: errorMessage }
-      }).catch(() => {
-      });
-    }
+    await recordJobFailure(job.tenantId, job.id, err);
   }
 }
 let workerLoopTimer = null;
@@ -275,7 +297,7 @@ async function retryEligibleFailedJobs() {
     SET status = 'QUEUED',
         "requeuedAt" = NOW()
     WHERE status = 'RETRY_WAIT'
-      AND "nextRetryAt" <= NOW()
+      AND "nextRetryAt" <= (NOW() AT TIME ZONE 'UTC')
       AND dlq = false
     RETURNING id
   `;
@@ -486,7 +508,7 @@ async function resumeJob(tenantId, jobId) {
     (tx) => tx.platformJob.findUnique({ where: { id: jobId } })
   );
   if (!job) throw new Error("Job not found");
-  const checkpoint = job.result?.checkpoint;
+  const checkpoint = job.checkpoint ?? job.result?.checkpoint;
   const visitedUrls = checkpoint?.visitedUrls ?? [];
   const pendingQueue = checkpoint?.pendingQueue ?? [];
   if (pendingQueue.length === 0 && visitedUrls.length === 0) {
@@ -495,8 +517,8 @@ async function resumeJob(tenantId, jobId) {
   return enqueueJob(job.type, {
     ...job.config,
     tenantId,
-    skipUrls: visitedUrls.length > 0 ? visitedUrls : void 0,
-    resumeUrls: pendingQueue.length > 0 ? pendingQueue : void 0
+    skipUrls: [...job.config?.skipUrls ?? [], ...visitedUrls],
+    startUrls: pendingQueue.length > 0 ? pendingQueue : job.config?.startUrls
   });
 }
 async function updateJobProgress(tenantId, jobId, progress, step) {
@@ -863,19 +885,7 @@ async function processJob(jobId, config, opts) {
         skipUrls: [...config.skipUrls ?? [], ...checkpoint.visitedUrls ?? []]
       }).catch((enqErr) => logger.warn({ err: enqErr?.message }, "[crawl] auth-failure auto-segment enqueue failed (non-fatal)"));
     }
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({
-        where: { id: jobId },
-        data: {
-          status: aborted ? "CANCELLED" : "FAILED",
-          error: aborted ? "Stopped by user" : String(err),
-          finishedAt: /* @__PURE__ */ new Date(),
-          ...aborted ? {} : { nextRetryAt: new Date(Date.now() + 6e4) },
-          ...checkpoint ? { result: { checkpoint } } : {}
-        }
-      })
-    );
+    await recordJobFailure(tenantId, jobId, err, checkpoint, { isAborted: aborted });
     if (!aborted) {
       notifyAdmins(tenantId, {
         type: "crawl.failed",
@@ -952,10 +962,7 @@ async function processApiTestsJob(jobId, config) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err) {
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date(), nextRetryAt: new Date(Date.now() + 6e4) } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processSecurityTestsJob(jobId, config) {
@@ -987,10 +994,7 @@ async function processSecurityTestsJob(jobId, config) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err) {
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date(), nextRetryAt: new Date(Date.now() + 6e4) } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processIntegrationTestsJob(jobId, config) {
@@ -1022,10 +1026,7 @@ async function processIntegrationTestsJob(jobId, config) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
   } catch (err) {
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date(), nextRetryAt: new Date(Date.now() + 6e4) } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processFigmaImportJob(jobId, config) {
@@ -1140,10 +1141,7 @@ async function processFigmaImportJob(jobId, config) {
       }
     });
   } catch (err) {
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processNativeCrawlJob(jobId, config) {
@@ -1234,11 +1232,7 @@ async function processNativeCrawlJob(jobId, config) {
       }
     });
   } catch (err) {
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processGenerateTestsFromReqsJob(jobId, config) {
@@ -1248,7 +1242,7 @@ async function processGenerateTestsFromReqsJob(jobId, config) {
     const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import("@detiq/agents");
     const llmCfg = await getTaskAgentLLMConfig(tenantId, "testGeneration");
     const all = await AppBrain.listRequirements(tenantId, projectId, {});
-    const reqs = reqIds?.length ? all.filter((r) => reqIds.includes(r.reqId)) : all.filter((r) => r.status === "unbuilt" || r.status === "pending");
+    let reqs = reqIds?.length ? all.filter((r) => reqIds.includes(r.reqId)) : all.filter((r) => r.status === "unbuilt" || r.status === "pending");
     if (reqs.length === 0) {
       await updateJobProgress(tenantId, jobId, 100, "No pending requirements found");
       await withTenant(
@@ -1267,76 +1261,99 @@ async function processGenerateTestsFromReqsJob(jobId, config) {
       });
       resolvedSuiteId = suite.id;
     }
+    const existingJob = await prismaAdmin.platformJob.findUnique({
+      where: { id: jobId },
+      select: { checkpoint: true }
+    }).catch(() => null);
+    const completedReqIds = new Set(existingJob?.checkpoint?.completedReqIds ?? []);
+    if (completedReqIds.size > 0) {
+      reqs = reqs.filter((r) => !completedReqIds.has(r.reqId));
+    }
     await updateJobProgress(tenantId, jobId, 20, `Generating tests for ${reqs.length} requirements`);
-    const reqBlock = reqs.map((r) => [
-      `## ${r.reqId}: ${r.title}`,
-      `Priority: ${r.priority} | Type: ${r.type}`,
-      r.acceptanceCriteria?.length ? `Acceptance Criteria:
+    const VALID_KINDS = /* @__PURE__ */ new Set(["POSITIVE", "NEGATIVE", "EDGE", "STATE_TRANSITION", "ACCESSIBILITY", "SECURITY", "INTEGRATION", "E2E", "API", "EXPLORATORY", "UAT", "PERFORMANCE", "COMPATIBILITY", "L10N", "USABILITY", "LOAD", "UNIT", "MUTATION", "AB_TEST", "ALPHA_BETA", "SYSTEM", "SMOKE", "REGRESSION", "SANITY"]);
+    const VALID_PRIORITIES = /* @__PURE__ */ new Set(["CRITICAL", "HIGH", "MEDIUM", "LOW"]);
+    const CHUNK_SIZE = 5;
+    let totalCreated = existingJob?.checkpoint?.createdCount ?? 0;
+    let totalSkipped = existingJob?.checkpoint?.skippedCount ?? 0;
+    const finishedReqIds = Array.from(completedReqIds);
+    for (let i = 0; i < reqs.length; i += CHUNK_SIZE) {
+      const chunk = reqs.slice(i, i + CHUNK_SIZE);
+      const reqBlock = chunk.map((r) => [
+        `## ${r.reqId}: ${r.title}`,
+        `Priority: ${r.priority} | Type: ${r.type}`,
+        r.acceptanceCriteria?.length ? `Acceptance Criteria:
 ${r.acceptanceCriteria.map((c) => `- ${c}`).join("\n")}` : "",
-      r.businessRules?.length ? `Business Rules:
+        r.businessRules?.length ? `Business Rules:
 ${r.businessRules.map((b) => `- ${b}`).join("\n")}` : ""
-    ].filter(Boolean).join("\n")).join("\n\n");
-    const system = `You are ZeTa-AI's test case generator. Convert requirements into structured test cases.
+      ].filter(Boolean).join("\n")).join("\n\n");
+      const system = `You are ZeTa-AI's test case generator. Convert requirements into structured test cases.
 Return ONLY valid JSON: { "cases": [ { "title": string, "kind": string, "priority": string, "preconditions": string, "steps": [{"action": string, "data": string, "expected": string}], "expectedResult": string, "tags": string[], "reqRef": string } ] }
 Kind must be one of: POSITIVE, NEGATIVE, EDGE, SECURITY, ACCESSIBILITY, INTEGRATION, E2E, SMOKE, REGRESSION.
 Priority must be one of: CRITICAL, HIGH, MEDIUM, LOW.
 Generate 1-3 test cases per requirement (positive + negative + edge if applicable).
 Each test case must reference its reqRef (the requirement ID like REQ-001).`;
-    const user = `Generate test cases for these requirements:
+      const user = `Generate test cases for these requirements:
 
 ${reqBlock}`;
-    let parsed = { cases: [] };
-    try {
-      const reqSeed = deterministicSeed(`req-cases:${reqs.map((r) => r.reqId).sort().join("|").slice(0, 300)}`);
-      const { text, usage } = await askTracked(system, user, llmCfg, void 0, void 0, { temperature: 0, seed: reqSeed });
-      if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: "generate_tests_from_reqs", label: "test_case_generation", usage }).catch(() => {
+      let parsed = { cases: [] };
+      try {
+        const reqSeed = deterministicSeed(`req-cases:${chunk.map((r) => r.reqId).sort().join("|").slice(0, 300)}`);
+        const { text, usage } = await askTracked(system, user, llmCfg, void 0, void 0, { temperature: 0, seed: reqSeed });
+        if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: "generate_tests_from_reqs", label: "test_case_generation", usage }).catch(() => {
+        });
+        const json = text.match(/\{[\s\S]*\}/)?.[0];
+        if (json) parsed = JSON.parse(json);
+      } catch (e) {
+        throw new Error("AI failed to generate test cases: " + (e.message ?? "unknown"));
+      }
+      const normalized = (parsed.cases ?? []).map((c) => ({
+        ref: c.reqRef ?? null,
+        traceTo: c.reqRef ?? null,
+        title: c.title || "Untitled",
+        kind: VALID_KINDS.has((c.kind ?? "").toUpperCase()) ? c.kind.toUpperCase() : "POSITIVE",
+        priority: VALID_PRIORITIES.has((c.priority ?? "").toUpperCase()) ? c.priority.toUpperCase() : "MEDIUM",
+        preconditions: c.preconditions ?? "",
+        steps: Array.isArray(c.steps) ? c.steps : [],
+        expectedResult: c.expectedResult ?? "",
+        tags: Array.isArray(c.tags) ? c.tags : []
+      }));
+      const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
+        tenantId,
+        projectId,
+        resolvedSuiteId,
+        normalized,
+        { duplicateMode: "skip" }
+      );
+      totalCreated += created.length;
+      totalSkipped += skipped ?? 0;
+      const coveredReqIds = new Set(normalized.map((c) => c.ref).filter(Boolean));
+      await Promise.all(
+        chunk.filter((r) => coveredReqIds.has(r.reqId)).map((r) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: "covered" }).catch(() => {
+        }))
+      );
+      for (const r of chunk) finishedReqIds.push(r.reqId);
+      await prismaAdmin.platformJob.update({
+        where: { id: jobId },
+        data: {
+          checkpoint: { completedReqIds: finishedReqIds, createdCount: totalCreated, skippedCount: totalSkipped },
+          checkpointAt: /* @__PURE__ */ new Date()
+        }
+      }).catch(() => {
       });
-      const json = text.match(/\{[\s\S]*\}/)?.[0];
-      if (json) parsed = JSON.parse(json);
-    } catch (e) {
-      throw new Error("AI failed to generate test cases: " + (e.message ?? "unknown"));
+      const progressPct = Math.round(20 + (i + chunk.length) / reqs.length * 75);
+      await updateJobProgress(tenantId, jobId, progressPct, `Processed ${finishedReqIds.length}/${reqs.length} requirements (${totalCreated} cases generated)`);
     }
-    await updateJobProgress(tenantId, jobId, 70, `Saving ${parsed.cases?.length ?? 0} test cases`);
-    const VALID_KINDS = /* @__PURE__ */ new Set(["POSITIVE", "NEGATIVE", "EDGE", "STATE_TRANSITION", "ACCESSIBILITY", "SECURITY", "INTEGRATION", "E2E", "API", "EXPLORATORY", "UAT", "PERFORMANCE", "COMPATIBILITY", "L10N", "USABILITY", "LOAD", "UNIT", "MUTATION", "AB_TEST", "ALPHA_BETA", "SYSTEM", "SMOKE", "REGRESSION", "SANITY"]);
-    const VALID_PRIORITIES = /* @__PURE__ */ new Set(["CRITICAL", "HIGH", "MEDIUM", "LOW"]);
-    const normalized = (parsed.cases ?? []).map((c) => ({
-      ref: c.reqRef ?? null,
-      traceTo: c.reqRef ?? null,
-      title: c.title || "Untitled",
-      kind: VALID_KINDS.has((c.kind ?? "").toUpperCase()) ? c.kind.toUpperCase() : "POSITIVE",
-      priority: VALID_PRIORITIES.has((c.priority ?? "").toUpperCase()) ? c.priority.toUpperCase() : "MEDIUM",
-      preconditions: c.preconditions ?? "",
-      steps: Array.isArray(c.steps) ? c.steps : [],
-      expectedResult: c.expectedResult ?? "",
-      tags: Array.isArray(c.tags) ? c.tags : []
-    }));
-    const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
-      tenantId,
-      projectId,
-      resolvedSuiteId,
-      normalized,
-      { duplicateMode: "skip" }
-    );
-    const coveredReqIds = new Set(normalized.map((c) => c.ref).filter(Boolean));
-    await Promise.all(
-      reqs.filter((r) => coveredReqIds.has(r.reqId)).map((r) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: "covered" }).catch(() => {
-      }))
-    );
     await withTenant(
       tenantId,
       (tx) => tx.platformJob.update({
         where: { id: jobId },
-        data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { created: created.length, skipped, suiteId: resolvedSuiteId }, progress: 100 }
+        data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { created: totalCreated, skipped: totalSkipped, suiteId: resolvedSuiteId }, progress: 100 }
       })
     );
   } catch (e) {
     const msg = e?.message ?? "Unknown error";
     logger.error({ jobId, err: msg }, "processGenerateTestsFromReqsJob failed");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, e);
   }
 }
 async function processRunTestSuiteJob(jobId, config) {
@@ -1373,6 +1390,17 @@ async function processRunTestSuiteJob(jobId, config) {
     let passed = 0;
     let failed = 0;
     const runResults = [];
+    const existingJob = await prismaAdmin.platformJob.findUnique({
+      where: { id: jobId },
+      select: { checkpoint: true }
+    }).catch(() => null);
+    const completedCaseIds = new Set(existingJob?.checkpoint?.completedCaseIds ?? []);
+    if (completedCaseIds.size > 0) {
+      testCases = testCases.filter((tc) => !completedCaseIds.has(tc.id));
+      passed = existingJob?.checkpoint?.passed ?? 0;
+      failed = existingJob?.checkpoint?.failed ?? 0;
+    }
+    const allFinishedCaseIds = Array.from(completedCaseIds);
     for (let i = 0; i < testCases.length; i += TEST_EXEC_CONCURRENCY) {
       const batch = testCases.slice(i, i + TEST_EXEC_CONCURRENCY);
       await Promise.all(batch.map(async (tc) => {
@@ -1435,6 +1463,15 @@ async function processRunTestSuiteJob(jobId, config) {
           });
         }
       }));
+      for (const tc of batch) allFinishedCaseIds.push(tc.id);
+      await prismaAdmin.platformJob.update({
+        where: { id: jobId },
+        data: {
+          checkpoint: { completedCaseIds: allFinishedCaseIds, passed, failed },
+          checkpointAt: /* @__PURE__ */ new Date()
+        }
+      }).catch(() => {
+      });
       await updateJobProgress(
         tenantId,
         jobId,
@@ -1470,11 +1507,7 @@ async function processRunTestSuiteJob(jobId, config) {
     logger.error({ jobId, testRunId, err: msg }, "processRunTestSuiteJob failed");
     await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => {
     });
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processGenerateSkillFilesJob(jobId, config) {
@@ -1515,10 +1548,7 @@ async function processGenerateSkillFilesJob(jobId, config) {
       (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date() } })
     );
   } catch (err) {
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processDataRetentionPurge() {
@@ -1751,17 +1781,7 @@ async function processDiscoverJob(jobId, config, opts) {
     }).catch(console.error);
   } catch (err) {
     logger.error({ err: err?.message, jobId }, "[discover] Job failed");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({
-        where: { id: jobId },
-        data: {
-          status: err?.message === "CRAWL_ABORTED" ? "CANCELLED" : "FAILED",
-          result: { error: err?.message ?? String(err) },
-          finishedAt: /* @__PURE__ */ new Date()
-        }
-      })
-    ).catch(console.error);
+    await recordJobFailure(tenantId, jobId, err);
   } finally {
     clearInterval(heartbeatTimer);
     runningControllers.delete(jobId);
@@ -1854,13 +1874,7 @@ async function processMonitorJob(jobId, config, opts) {
       })
     );
   } catch (err) {
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({
-        where: { id: jobId },
-        data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() }
-      })
-    );
+    await recordJobFailure(tenantId, jobId, err);
   } finally {
     runningControllers.delete(jobId);
   }
@@ -2013,8 +2027,13 @@ async function processGenerateScreenTestsJob(jobId, config) {
     await updateJobProgress(tenantId, jobId, 10, "Generating test cases for screen");
     const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null);
     const deviceProfiles = crawlCfg?.deviceProfiles ?? ["DESKTOP"];
+    const existingAiCases = await withTenant(
+      tenantId,
+      (tx) => tx.testCase.count({ where: { tenantId, screenId, source: "AI_GENERATED" } })
+    );
+    const generationMode = existingAiCases > 0 ? "replace" : void 0;
     for (const viewport of deviceProfiles) {
-      await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport });
+      await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport, ...generationMode ? { mode: generationMode } : {} });
     }
     await updateJobProgress(tenantId, jobId, 80, "Test cases generated");
     if (triggerAlignedScripts && repoConnectionId) {
@@ -2050,11 +2069,7 @@ async function processGenerateScreenTestsJob(jobId, config) {
   } catch (err) {
     const msg = err?.message ?? String(err);
     logger.error({ jobId, screenId, err: msg }, "processGenerateScreenTestsJob failed");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: msg.slice(0, 500), finishedAt: /* @__PURE__ */ new Date() } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 function captureStepText(step) {
@@ -2288,13 +2303,8 @@ async function processCaptureStepsJob(jobId, config) {
     );
   } catch (err) {
     const error = String(err?.message ?? err);
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error, finishedAt: /* @__PURE__ */ new Date(), heartbeatAt: /* @__PURE__ */ new Date() } })
-    ).catch(() => {
-    });
     logger.error({ jobId, testCaseId, err: error }, "processCaptureStepsJob failed");
-    throw err;
+    await recordJobFailure(tenantId, jobId, err);
   } finally {
     await browser?.close?.().catch(() => {
     });
@@ -2430,11 +2440,7 @@ async function processMultiRoleCrawlJob(jobId, config) {
   } catch (err) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, "[multi-role-crawl] failed");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: msg, finishedAt: /* @__PURE__ */ new Date() } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processRoleCrawlJob(jobId, config) {
@@ -2522,11 +2528,7 @@ async function processRoleCrawlJob(jobId, config) {
   } catch (err) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, "[role-crawl] outer failure");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: msg, finishedAt: /* @__PURE__ */ new Date() } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processGenerateScriptJob(jobId, config) {
@@ -2671,11 +2673,7 @@ async function processArtifactIntelJob(jobId, config) {
       (tx) => tx.projectArtifact.update({ where: { id: artifactId }, data: { status: "ERROR", summary: "AI analysis failed" } })
     ).catch(() => {
     });
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
     queueImportedArtifactSkillRefresh(tenantId, projectId, "artifact-intel-failed");
   }
 }
@@ -3183,11 +3181,7 @@ async function processRepoScanJob(jobId, config) {
       (tx) => tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: "FAILED", scanError: msg.slice(0, 500) } })
     ).catch(() => {
     });
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processBatchGenerateAlignedJob(jobId, config) {
@@ -3199,7 +3193,6 @@ async function processBatchGenerateAlignedJob(jobId, config) {
       (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
     );
     if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
-    const { decrypt: decrypt2 } = await import("@detiq/core");
     const {
       generateScriptAligned,
       deriveRepoTargetPath,
@@ -3298,11 +3291,7 @@ async function processBatchGenerateAlignedJob(jobId, config) {
   } catch (err) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, "[batch-generate-aligned] failed");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 async function processCreatePRJob(jobId, config) {
@@ -3408,11 +3397,7 @@ async function processCreatePRJob(jobId, config) {
   } catch (err) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, "[create-pr] failed");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 function buildPRBody(scripts, projectId) {
@@ -3436,7 +3421,7 @@ function buildPRBody(scripts, projectId) {
   return lines.join("\n");
 }
 async function processNewPagesPipelineJob(jobId, config) {
-  const { tenantId, projectId, crawlJobId, repoConnectionId, autoPR = false } = config;
+  const { tenantId, projectId, crawlJobId: _crawlJobId, repoConnectionId, autoPR = false } = config;
   try {
     await updateJobProgress(tenantId, jobId, 5, "Detecting new screens");
     const conn = await withTenant(
@@ -3504,11 +3489,7 @@ async function processNewPagesPipelineJob(jobId, config) {
   } catch (err) {
     const msg = String(err?.message ?? err);
     logger.error({ jobId, err: msg }, "[new-pages-pipeline] failed");
-    await withTenant(
-      tenantId,
-      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
-    ).catch(() => {
-    });
+    await recordJobFailure(tenantId, jobId, err);
   }
 }
 export {
@@ -3529,6 +3510,7 @@ export {
   processDataRetentionPurge,
   processJob,
   promoteToDeadLetterQueue,
+  recordJobFailure,
   requeueOrphanedJobs,
   requeueStaleRunningJobs,
   rerunJob,