@detiq/crawler 1.1.25 → 1.1.27

package/src/job-queue.ts

@@ -18,40 +18,18 @@ import crypto from 'node:crypto';
 import { withTenant, prismaAdmin } from '@detiq/database';
 import { crawlJobsTotal, crawlJobDurationSeconds } from '@detiq/core';
 import { logger } from './logger.js';
+import { AppBrain, getProjectStorageConfig, buildArtifactKey, uploadArtifact, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall, downloadBufferFromTenantStorage, dispatchToTenantAdmins } from '@detiq/app-brain';
 
 async function notifyAdmins(tenantId: string, data: {
   type: string; title: string; body: string; metadata?: Record<string, unknown>
 }) {
   try {
-    // Notification/Membership both have FORCE ROW LEVEL SECURITY — a bare
-    // prisma call with no app.current_tenant set gets silently filtered to
-    // zero rows (SELECT) or rejected (INSERT), not an error, so this call
-    // site had been a complete no-op with no visible failure until fixed.
-    const members = await withTenant(tenantId, (tx) =>
-      tx.membership.findMany({
-        where: { tenantId, role: { in: ['OWNER', 'ADMIN'] } },
-        select: { userId: true },
-      })
-    ) as any[];
-    await withTenant(tenantId, (tx) =>
-      tx.notification.createMany({
-        data: members.map((m: { userId: string }) => ({
-          id: `notif_${crypto.randomUUID()}`,
-          tenantId,
-          userId: m.userId,
-          type: data.type,
-          title: data.title,
-          body: data.body,
-          metadata: data.metadata as any,
-        })),
-      })
-    );
+    await dispatchToTenantAdmins(tenantId, data as any);
   } catch (e) {
     console.error('[notify] failed:', e);
   }
 }
-import { AppBrain, getProjectStorageConfig, buildArtifactKey, uploadArtifact, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from '@detiq/app-brain';
-import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, refreshTestExecutionHistorySkillFile, refreshVisualRegressionHistorySkillFile, refreshLocatorStrategyWithHealingData, refreshFieldCatalogSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
+import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, refreshTestExecutionHistorySkillFile, refreshVisualRegressionHistorySkillFile, refreshLocatorStrategyWithHealingData, refreshFieldCatalogSkillFile, detectFieldsFromHtml, mergeIntoCatalog, generateArtifactIntelligence } from '@detiq/agents';
 import type { ExecutionStep } from '@detiq/agents';
 import { crawlProject, domFallbackElements } from './crawler.js';
 import { httpCrawl } from './http-engine.js';
@@ -67,6 +45,70 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import os from 'node:os';
 
+// ── GAP-14/15: Structured failure classification with per-type backoff ────────────────
+
+export enum FailureType {
+  TRANSIENT = 'TRANSIENT',           // network timeout, rate limit, 429, 503
+  INFRASTRUCTURE = 'INFRASTRUCTURE', // DB connection, OOM, worker crash
+  LOGICAL = 'LOGICAL',               // bad input, business rule violation, 400/422
+  FATAL = 'FATAL',                   // unrecoverable — bad config, missing secrets
+}
+
+interface BackoffConfig {
+  maxRetries: number;
+  baseDelayMs: number;
+  multiplier: number;
+  maxDelayMs: number;
+}
+
+const BACKOFF_BY_TYPE: Record<FailureType, BackoffConfig> = {
+  [FailureType.TRANSIENT]:       { maxRetries: 5, baseDelayMs: 30_000,  multiplier: 2, maxDelayMs: 300_000  },
+  [FailureType.INFRASTRUCTURE]:  { maxRetries: 3, baseDelayMs: 60_000,  multiplier: 3, maxDelayMs: 600_000  },
+  [FailureType.LOGICAL]:         { maxRetries: 1, baseDelayMs: 120_000, multiplier: 1, maxDelayMs: 120_000  },
+  [FailureType.FATAL]:           { maxRetries: 0, baseDelayMs: 0,       multiplier: 1, maxDelayMs: 0        },
+};
+
+export function classifyFailure(error: string | Error | unknown): FailureType {
+  const msg = (error instanceof Error ? error.message : String(error)).toLowerCase();
+
+  // Fatal — bad configuration, missing secrets
+  if (msg.includes('master_enc_key') || msg.includes('missing secret') || msg.includes('invalid configuration')) {
+    return FailureType.FATAL;
+  }
+
+  // Infrastructure — OOM, DB, worker crashes
+  if (
+    msg.includes('out of memory') || msg.includes('heap out') ||
+    msg.includes('connection refused') || msg.includes('econnrefused') ||
+    msg.includes('prisma') || msg.includes('database') ||
+    msg.includes('worker exited') || msg.includes('sigkill')
+  ) {
+    return FailureType.INFRASTRUCTURE;
+  }
+
+  // Logical — validation, auth, bad input
+  if (
+    msg.includes('validation') || msg.includes('invalid input') ||
+    msg.includes('not found') || msg.includes('unauthorized') ||
+    msg.includes('bad request') || msg.includes('400') || msg.includes('422') ||
+    msg.includes('403')
+  ) {
+    return FailureType.LOGICAL;
+  }
+
+  // Default: TRANSIENT (network, timeout, rate limit, 429, 503)
+  return FailureType.TRANSIENT;
+}
+
+function computeNextRetryAt(failureType: FailureType, retryCount: number): Date | null {
+  const config = BACKOFF_BY_TYPE[failureType];
+  if (retryCount >= config.maxRetries) return null; // exceeded max — go to DLQ
+  const delayMs = Math.min(config.baseDelayMs * Math.pow(config.multiplier, retryCount), config.maxDelayMs);
+  return new Date(Date.now() + delayMs);
+}
+
+// ────────────────────────────────────────────────────────────────────────────────
+
 // RFC 6238 TOTP — no external dependency, uses Node built-in crypto.
 export function generateTOTP(secret: string): string {
   const BASE32 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
@@ -227,8 +269,24 @@ const runningControllers = new Map<string, AbortController>();
 
 const WORKER_ID = `${os.hostname()}:${process.pid}`;
 const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
+const MAX_CONCURRENT_PER_TENANT = Number(process.env.MAX_CONCURRENT_PER_TENANT) || 4;
 const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15_000;
 const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
+
+// GAP-12: true active-execution semaphore — counts jobs currently executing
+// (claim→finish), not poll slots. A single poll loop checks this before every
+// claim so the cap is never exceeded even across overlapping ticks.
+let activeExecutions = 0;
+
+function withSemaphore<T>(fn: () => Promise<T>): Promise<T> {
+  activeExecutions++;
+  return fn().finally(() => { activeExecutions--; });
+}
+
+export function getActiveExecutionCount(): number {
+  return activeExecutions;
+}
+
 // Max simultaneous DISCOVER jobs — each launches a Playwright browser and holds
 // DB connections, so running many concurrently exhausts resources. Default 2.
 const MAX_CONCURRENT_DISCOVER = Number(process.env.MAX_CONCURRENT_DISCOVER) || 2;
@@ -252,6 +310,7 @@ export async function claimSpecificJob(jobId: string): Promise<ClaimedJob | null
     UPDATE "PlatformJob"
     SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
     WHERE id = ${jobId} AND status = 'QUEUED'
+      AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
     RETURNING id, "tenantId" AS "tenantId", "projectId" AS "projectId", type, config
   `;
   return rows[0] ?? null;
@@ -267,13 +326,20 @@ export async function claimNextQueuedJob(): Promise<ClaimedJob | null> {
     UPDATE "PlatformJob"
     SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
     WHERE id = (
-      SELECT id FROM "PlatformJob"
+      SELECT id FROM "PlatformJob" p
       WHERE status = 'QUEUED'
+        AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
         AND (
           type != 'DISCOVER'
           OR (SELECT COUNT(*) FROM "PlatformJob" WHERE status = 'RUNNING' AND type = 'DISCOVER') < ${maxDiscover}
         )
-      ORDER BY "queuedAt" ASC
+        AND (
+          SELECT COUNT(*) FROM "PlatformJob" p2
+          WHERE p2.status = 'RUNNING'
+            AND p2."tenantId" = p."tenantId"
+            AND p2.inline = false
+        ) < ${MAX_CONCURRENT_PER_TENANT}
+      ORDER BY p."priority" DESC, p."queuedAt" ASC
       FOR UPDATE SKIP LOCKED
       LIMIT 1
     )
@@ -376,36 +442,77 @@ export async function dispatchClaimedJob(job: ClaimedJob): Promise<void> {
     stopTimer();
     crawlJobsTotal.inc({ type: job.type, status: 'failed' });
     logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, '[job-worker] job threw uncaught error');
+    const errorMessage = String(err?.message ?? err);
+    const failureType = classifyFailure(err);
+    // Read current retryCount before incrementing so computeNextRetryAt uses the right base.
+    const jobRow = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { retryCount: true, result: true } }).catch(() => null);
+    const currentRetry = jobRow?.retryCount ?? 0;
+    const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
     await prismaAdmin.platformJob.update({
       where: { id: job.id },
-      data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date(), nextRetryAt: new Date(Date.now() + 60_000) },
+      data: {
+        status: nextRetryAt ? 'RETRY_WAIT' : 'FAILED',
+        error: errorMessage,
+        finishedAt: new Date(),
+        heartbeatAt: null,
+        workerId: null,
+        nextRetryAt,
+        retryCount: { increment: 1 },
+        failureType,
+        // Store failureType in result JSON for observability (backward compat)
+        result: { ...(jobRow?.result as any ?? {}), failureType } as any,
+      },
     }).catch(() => { });
+    // If fatal or exceeded retries — promote to DLQ immediately
+    if (!nextRetryAt || failureType === FailureType.FATAL) {
+      await prismaAdmin.platformJob.update({
+        where: { id: job.id },
+        data: { dlq: true, dlqAt: new Date(), dlqReason: errorMessage },
+      }).catch(() => { });
+    }
   }
 }
 
 let workerLoopTimer: ReturnType<typeof setInterval> | null = null;
+// GAP-12: signals the semaphore-based poll loop to exit gracefully.
+let workerShuttingDown = false;
 
-/** Re-promotes FAILED jobs to QUEUED if they are eligible for retry based on per-type limits. */
+/** Re-promotes FAILED jobs to RETRY_WAIT if they are eligible for retry based on per-type limits.
+ *  RETRY_WAIT makes the waiting period visible in the UI. A separate sweep then promotes
+ *  RETRY_WAIT → QUEUED once nextRetryAt <= NOW(). */
 export async function retryEligibleFailedJobs(): Promise<void> {
+  // Legacy path: jobs that failed before failure-type classification was added.
+  // New failures go FAILED→RETRY_WAIT directly in the job error handler (dispatchClaimedJob).
+  // This only picks up old FAILED rows that have no nextRetryAt set yet.
+  // Phase 1: FAILED → RETRY_WAIT (GAP-07: visible waiting state)
   const rows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
     UPDATE "PlatformJob"
-    SET status = 'QUEUED',
+    SET status = 'RETRY_WAIT',
         "retryCount" = "retryCount" + 1,
         "nextRetryAt" = NOW() + (INTERVAL '1 minute' * POWER(2, "retryCount")),
         "workerId" = NULL
     WHERE status = 'FAILED'
       AND dlq = false
-      AND "nextRetryAt" <= NOW()
-      AND "retryCount" < CASE
-        WHEN type IN ('CRAWL', 'NATIVE_CRAWL') THEN 3
-        WHEN type IN ('GENERATE_TESTS', 'IMPORT_TESTS', 'RUN_TEST_SUITE', 'VISUAL_REGRESSION') THEN 2
-        WHEN type IN ('GENERATE_SCRIPT', 'GENERATE_DATA', 'GENERATE_TEST_PLAN') THEN 0
-        ELSE 1
-      END
+      AND "nextRetryAt" IS NULL
+      AND "retryCount" < 5
     RETURNING id
   `;
   if (rows.length > 0) {
-    logger.info({ jobIds: rows.map((r: any) => r.id) }, '[job-worker] requeued failed job(s) for retry');
+    logger.info({ jobIds: rows.map((r: any) => r.id) }, '[job-worker] moved failed job(s) to RETRY_WAIT');
+  }
+
+  // Phase 2: RETRY_WAIT → QUEUED when the wait period has elapsed (GAP-07)
+  const readyRows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
+    UPDATE "PlatformJob"
+    SET status = 'QUEUED',
+        "requeuedAt" = NOW()
+    WHERE status = 'RETRY_WAIT'
+      AND "nextRetryAt" <= NOW()
+      AND dlq = false
+    RETURNING id
+  `;
+  if (readyRows.length > 0) {
+    logger.info({ jobIds: readyRows.map((r: any) => r.id) }, '[job-worker] promoted RETRY_WAIT job(s) to QUEUED');
   }
 }
 
@@ -414,6 +521,7 @@ export async function promoteToDeadLetterQueue(): Promise<void> {
   const rows = await prismaAdmin.$queryRaw<Array<{ id: string }>>`
     UPDATE "PlatformJob"
     SET dlq = true,
+        "dlqAt" = NOW(),
         "dlqReason" = error
     WHERE status = 'FAILED'
       AND dlq = false
@@ -444,7 +552,8 @@ export function startJobWorkerLoop(opts?: { pollIntervalMs?: number; concurrency
   const concurrency = opts?.concurrency ?? WORKER_CONCURRENCY;
   logger.info({ workerId: WORKER_ID, pollIntervalMs, concurrency }, '[job-worker] starting');
 
-  const tick = async () => {
+  // GAP-12: maintenance sweep stays on a fixed interval (unchanged).
+  const sweepTick = async () => {
     try {
       await retryEligibleFailedJobs();
       await promoteToDeadLetterQueue();
@@ -452,33 +561,71 @@ export function startJobWorkerLoop(opts?: { pollIntervalMs?: number; concurrency
     } catch (err) {
       logger.error({ err: String(err) }, '[job-worker] stale/retry job sweep failed');
     }
-    try {
-      const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: 'queue.paused' } });
-      if (pauseRow?.value === 'true') {
+  };
+  workerLoopTimer = setInterval(sweepTick, pollIntervalMs);
+
+  // GAP-12: single continuous poll loop — checks the semaphore before every
+  // claim so activeExecutions never exceeds WORKER_CONCURRENCY regardless of
+  // how long individual jobs take.  Jobs are non-blocking: withSemaphore(...)
+  // is NOT awaited here, so the loop immediately loops back and can claim the
+  // next job while previous ones are still running.
+  //
+  // NOTE (GAP-20): requeueOrphanedJobs() is intentionally NOT called at startup here.
+  // Calling it at startup is unsafe in multi-replica deployments — replica A would steal
+  // RUNNING jobs from replica B (which is still alive and processing them), causing
+  // double-execution. The stale-job sweep (requeueStaleRunningJobs) handles recovery
+  // safely via heartbeat timeout. requeueOrphanedJobs() remains available for admin use.
+  workerShuttingDown = false;
+
+  const pollLoop = async () => {
+    void sweepTick(); // run one sweep immediately on startup
+    while (!workerShuttingDown) {
+      if (activeExecutions >= concurrency) {
+        // All slots busy — wait before checking again without hammering the DB.
+        await new Promise(resolve => setTimeout(resolve, 500));
+        continue;
+      }
+
+      let paused = false;
+      try {
+        const pauseRow = await prismaAdmin.systemConfig.findUnique({ where: { key: 'queue.paused' } });
+        paused = pauseRow?.value === 'true';
+      } catch {
+        // ignore — treat as unpaused
+      }
+
+      if (paused) {
         logger.info('[job-worker] queue is paused — skipping claim');
-      } else {
-        for (let i = 0; i < concurrency; i++) {
-          const job = await claimNextQueuedJob();
-          if (!job) break;
-          void dispatchClaimedJob(job); // not awaited — runs concurrently while the loop keeps polling
-        }
+        await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+        continue;
       }
-    } catch (err) {
-      logger.error({ err: String(err) }, '[job-worker] claim loop failed');
+
+      let job: ClaimedJob | null = null;
+      try {
+        job = await claimNextQueuedJob();
+      } catch (err) {
+        logger.error({ err: String(err) }, '[job-worker] claim failed');
+      }
+
+      if (!job) {
+        // Nothing queued — back off before polling again.
+        await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+        continue;
+      }
+
+      // Non-blocking: fire and forget into semaphore — poll loop continues immediately.
+      withSemaphore(() => dispatchClaimedJob(job!)).catch(err => {
+        logger.error({ err, jobId: job!.id }, '[worker] unhandled job error');
+      });
     }
   };
 
-  workerLoopTimer = setInterval(tick, pollIntervalMs);
-  // On startup: immediately requeue jobs orphaned by the previous process instance.
-  // This gives instant recovery rather than waiting STALE_JOB_TIMEOUT_MINUTES (30 min).
-  requeueOrphanedJobs().catch((err) =>
-    logger.error({ err: String(err) }, '[job-worker] startup orphan sweep failed')
-  );
-  void tick();
+  void pollLoop();
 }
 
 /** Stops the poll loop. Mainly for tests. */
 export function stopJobWorkerLoop(): void {
+  workerShuttingDown = true; // GAP-12: signal continuous poll loop to exit
   if (workerLoopTimer) clearInterval(workerLoopTimer);
   workerLoopTimer = null;
 }
@@ -491,31 +638,50 @@ export async function createTrackedJob(
   const name = makeJobName(type);
   const job = await withTenant(config.tenantId, (tx) =>
     tx.platformJob.create({
-      data: { tenantId: config.tenantId, projectId: config.projectId, type, name, status: 'RUNNING', startedAt: new Date(), config: config as any },
+      data: { tenantId: config.tenantId, projectId: config.projectId, type, name, status: 'RUNNING', startedAt: new Date(), inline: true, config: config as any },
     })
   ) as { id: string };
   return job.id;
 }
 
 /** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
- * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
+ * GAP-14/15: Uses failure-type classification for per-type backoff on FAILED. */
 export async function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void> {
-  const nextRetryAt = error ? new Date(Date.now() + 60_000) : undefined;
+  if (!error) {
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), progress: 100 } })
+    ).catch(() => { });
+    return;
+  }
+  const failureType = classifyFailure(error);
+  const jobRow = await prismaAdmin.platformJob.findUnique({ where: { id: jobId }, select: { retryCount: true, result: true } }).catch(() => null);
+  const currentRetry = jobRow?.retryCount ?? 0;
+  const nextRetryAt = computeNextRetryAt(failureType, currentRetry);
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
       data: {
-        status: error ? 'FAILED' : 'DONE',
+        status: nextRetryAt ? 'RETRY_WAIT' : 'FAILED',
         finishedAt: new Date(),
-        ...(error ? { error, nextRetryAt } : { progress: 100 }),
+        error,
+        nextRetryAt,
+        retryCount: { increment: 1 },
+        failureType,
+        result: { ...(jobRow?.result as any ?? {}), failureType } as any,
       },
     })
   ).catch(() => { });
+  if (!nextRetryAt || failureType === FailureType.FATAL) {
+    await prismaAdmin.platformJob.update({
+      where: { id: jobId },
+      data: { dlq: true, dlqAt: new Date(), dlqReason: error },
+    }).catch(() => { });
+  }
 }
 
 /** Enqueue a generic platform job — returns the job ID immediately. */
 export async function enqueueJob(type: string, config: any): Promise<string> {
-  const { tenantId, projectId } = config;
+  const { tenantId, projectId, submittedBy } = config;
 
   const name = makeJobName(type);
   const job = await withTenant(tenantId, (tx) =>
@@ -527,6 +693,7 @@ export async function enqueueJob(type: string, config: any): Promise<string> {
         name,
         status: 'QUEUED',
         config: config as any,
+        ...(submittedBy ? { submittedBy } : {}),
       },
     })
   ) as { id: string };
@@ -580,7 +747,10 @@ export async function listJobs(tenantId: string, projectId: string) {
   });
 }
 
-/** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
+/** Cancel a RUNNING or QUEUED job.
+ *  Sets status to CANCEL_REQUESTED (GAP-08) so the worker's abort handler can
+ *  detect the request and set CANCELLED + finishedAt after the underlying work stops.
+ *  The in-process AbortController is also triggered for same-process jobs. */
 export async function cancelJob(tenantId: string, jobId: string): Promise<void> {
   // Abort in-memory controller if job is running in the same process
   const controller = runningControllers.get(jobId);
@@ -591,7 +761,7 @@ export async function cancelJob(tenantId: string, jobId: string): Promise<void>
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: 'CANCELLED', finishedAt: new Date(), heartbeatAt: null, workerId: null },
+      data: { status: 'CANCEL_REQUESTED' },
     })
   );
 }
@@ -636,20 +806,21 @@ export async function updateJobProgress(tenantId: string, jobId: string, progres
   try {
     await withTenant(tenantId, async (tx) => {
       // DB-flag cancellation: works for both inline and SQS backends.
-      // When a user cancels, status is set to 'CANCELLED' in DB.
+      // When a user cancels, status is set to CANCEL_REQUESTED (GAP-08).
       // This check fires at each progress update (every few seconds during crawl).
       const current = await tx.platformJob.findUnique({
         where: { id: jobId },
         select: { status: true },
       });
-      if (current?.status === 'CANCELLED') {
+      if (current?.status === 'CANCELLED' || current?.status === 'CANCEL_REQUESTED') {
         runningControllers.get(jobId)?.abort();
         runningControllers.delete(jobId);
         return; // abort signal propagates; crawlProject exits on next signal check
       }
       await tx.platformJob.update({
         where: { id: jobId },
-        data: { progress, result: { step } as any, heartbeatAt: new Date() },
+        // GAP-09: write currentStep as a proper column in addition to result.step (backward compat)
+        data: { progress, currentStep: step, result: { step } as any, heartbeatAt: new Date() },
       });
     });
   } catch { /* non-fatal — progress updates best-effort */ }
@@ -663,7 +834,7 @@ async function heartbeatJob(tenantId: string, jobId: string) {
         where: { id: jobId },
         select: { status: true },
       });
-      if (current?.status === 'CANCELLED') {
+      if (current?.status === 'CANCELLED' || current?.status === 'CANCEL_REQUESTED') {
         runningControllers.get(jobId)?.abort();
         runningControllers.delete(jobId);
         return;
@@ -800,6 +971,19 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
       storageConfig: storageConfigForCrawl,
       onCheckpoint: async (visitedUrls, pendingQueue) => {
         jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
+        // GAP-19: configurable checkpoint interval; GAP-04: persist to DB for crash recovery
+        const CHECKPOINT_INTERVAL = parseInt(process.env.CRAWL_CHECKPOINT_INTERVAL ?? '10');
+        if (visitedUrls.length % CHECKPOINT_INTERVAL === 0) {
+          try {
+            await prismaAdmin.platformJob.update({
+              where: { id: jobId },
+              data: {
+                checkpoint: { visitedUrls, pendingQueue } as any,
+                checkpointAt: new Date(),
+              },
+            });
+          } catch { /* non-fatal */ }
+        }
       },
     }, onProgress);
     jobCheckpoints.delete(jobId);
@@ -1482,7 +1666,7 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
 
     let parsed: { cases: any[] } = { cases: [] };
     try {
-      const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
+      const reqSeed = deterministicSeed(`req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
       const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
       if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => { });
       const json = text.match(/\{[\s\S]*\}/)?.[0];
@@ -1789,6 +1973,20 @@ export async function processDataRetentionPurge(): Promise<void> {
     },
   });
   logger.info({ deleted: sessionResult.count }, '[data-retention] AuthSession purged');
+
+  // PlatformJob: delete completed jobs older than 90 days (GAP-24)
+  const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000);
+  const deletedJobs = await prismaAdmin.platformJob.deleteMany({
+    where: {
+      status: { in: ['DONE', 'FAILED'] },
+      dlq: false,
+      inline: false,
+      finishedAt: { lt: ninetyDaysAgo },
+    },
+  });
+  if (deletedJobs.count > 0) {
+    logger.info({ count: deletedJobs.count }, '[data-retention] deleted old completed jobs');
+  }
 }
 
 // ── URL Discovery Job ─────────────────────────────────────────────────────────
@@ -2341,8 +2539,13 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
     const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null);
     const deviceProfiles: string[] = (crawlCfg as any)?.deviceProfiles ?? ['DESKTOP'];
 
+    const existingAiCases: number = await withTenant(tenantId, (tx) =>
+      (tx as any).testCase.count({ where: { tenantId, screenId, source: 'AI_GENERATED' } })
+    );
+    const generationMode = existingAiCases > 0 ? 'replace' : undefined;
+
     for (const viewport of deviceProfiles) {
-      await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport });
+      await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport, ...(generationMode ? { mode: generationMode } : {}) });
     }
 
     await updateJobProgress(tenantId, jobId, 80, 'Test cases generated');
@@ -2977,13 +3180,111 @@ async function processGenerateScriptJob(jobId: string, config: any): Promise<voi
   }
 }
 
+function queueImportedArtifactSkillRefresh(tenantId: string, projectId: string, reason: string): void {
+  setImmediate(() => {
+    import('@detiq/agents')
+      .then((agentsModule) => {
+        const agents = agentsModule as any;
+        return Promise.allSettled([
+          agents.refreshArtifactKnowledgeSkillFile(tenantId, projectId),
+          agents.refreshRequirementsCoverageSkillFile(tenantId, projectId),
+        ]);
+      })
+      .then((results) => {
+        results?.forEach((result) => {
+          if (result.status === 'rejected') {
+            console.warn(`[artifacts] skill refresh failed after ${reason}:`, result.reason?.message ?? result.reason);
+          }
+        });
+      })
+      .catch((e) => console.warn(`[artifacts] skill refresh failed after ${reason}:`, e?.message ?? e));
+  });
+}
+
 // Registers the type→processor dispatch table used by the durable worker (both the
 // enqueueJob() fast-path kick and startJobWorkerLoop()'s poll). Declared at module
 // bottom so it can reference every process*Job function by name; function
 // declarations are hoisted, so this could technically live anywhere in the file,
 // but keeping it here alongside the last definition keeps the mapping easy to audit
 // against the actual functions above.
+const CONTENT_DETECTED_DOC_TYPES = new Set(['FUNCTIONIZE_TC', 'SELENIUM_SCRIPT', 'ENV_CONFIG']);
+const AI_DOC_TYPES = new Set([
+  'REQUIREMENTS', 'USER_STORY', 'TEST_CASES', 'TEST_DATA', 'AUTOMATION_SCRIPT', 'API_SPEC', 'ARCHITECTURE', 'REPORT', 'OTHER',
+  'SPREADSHEET', 'DATASHEET',
+]);
+const AI_DOC_TYPE_ALIASES: Record<string, string> = {
+  REQUIREMENT: 'REQUIREMENTS', SPEC: 'REQUIREMENTS', SPECIFICATION: 'REQUIREMENTS',
+  USER_STORIES: 'USER_STORY', STORY: 'USER_STORY',
+  TEST_CASE: 'TEST_CASES', TESTCASES: 'TEST_CASES',
+  TESTDATA: 'TEST_DATA', SCRIPT: 'AUTOMATION_SCRIPT', AUTOMATION: 'AUTOMATION_SCRIPT',
+};
+function normalizeAiDocType(raw: unknown): string {
+  const key = String(raw ?? '').trim().toUpperCase().replace(/[\s-]+/g, '_');
+  const mapped = AI_DOC_TYPE_ALIASES[key] ?? key;
+  return AI_DOC_TYPES.has(mapped) ? mapped : 'OTHER';
+}
+
+async function processArtifactIntelJob(jobId: string, config: any) {
+  const { tenantId, projectId, artifactId, artifactName, isReprocess } = config;
+  try {
+    await updateJobProgress(tenantId, jobId, 10, 'Downloading artifact content');
+    
+    const artifact = await withTenant(tenantId, (tx) =>
+      (tx as any).projectArtifact.findUnique({ where: { id: artifactId }, select: { storageKey: true, docType: true } })
+    ) as any;
+    
+    if (!artifact || !artifact.storageKey) {
+      await updateJobProgress(tenantId, jobId, 100, 'Artifact deleted or missing content, skipping.');
+      await withTenant(tenantId, (tx) =>
+        (tx as any).platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date() } })
+      );
+      return;
+    }
+
+    const storageCfg = await getTenantStorageConfig(tenantId);
+    if (!storageCfg) {
+      throw new Error('Tenant storage configuration missing.');
+    }
+    const contentBuffer = await downloadBufferFromTenantStorage(storageCfg, artifact.storageKey);
+    const content = contentBuffer.toString('utf8');
+
+    await updateJobProgress(tenantId, jobId, 40, 'Generating intelligence');
+
+    const intel = await generateArtifactIntelligence(tenantId, projectId, artifactId, content, artifactName, jobId);
+    
+    const finalDocType = artifact.docType && CONTENT_DETECTED_DOC_TYPES.has(artifact.docType)
+      ? artifact.docType
+      : normalizeAiDocType(intel.docType);
+
+    if (finalDocType !== intel.docType) {
+      await withTenant(tenantId, (tx) =>
+        (tx as any).projectArtifact.update({ where: { id: artifactId }, data: { docType: finalDocType } })
+      ).catch((err: any) => console.warn(`[artifact-intel] docType fix-up failed for "${artifactName}":`, err?.message));
+      intel.docType = finalDocType;
+    }
+
+    await updateJobProgress(tenantId, jobId, 90, 'Saving results');
+    
+    await withTenant(tenantId, (tx) =>
+      (tx as any).platformJob.update({ where: { id: jobId }, data: { result: { artifactId, artifactName, summary: intel.summary, keywords: intel.keywords, docType: intel.docType }, status: 'DONE', finishedAt: new Date(), progress: 100 } })
+    );
+
+    queueImportedArtifactSkillRefresh(tenantId, projectId, isReprocess ? 'artifact-reprocess' : 'artifact-intel');
+  } catch (err: any) {
+    const msg = String(err?.message ?? err);
+    logger.error({ jobId, err: msg }, '[artifact-intel] failed');
+    await withTenant(tenantId, (tx) =>
+      (tx as any).projectArtifact.update({ where: { id: artifactId }, data: { status: 'ERROR', summary: 'AI analysis failed' } })
+    ).catch(() => { });
+    await withTenant(tenantId, (tx) =>
+      (tx as any).platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
+    ).catch(() => { });
+    queueImportedArtifactSkillRefresh(tenantId, projectId, 'artifact-intel-failed');
+  }
+}
+
 jobDispatchTable = {
+  ARTIFACT_INTEL: processArtifactIntelJob,
   CRAWL: processJob,
   CAPTURE_STEPS: processCaptureStepsJob,
   DISCOVER: processDiscoverJob,
@@ -3506,8 +3807,6 @@ async function processBatchGenerateAlignedJob(jobId: string, config: any) {
     ) as any;
     if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
 
-    const { decrypt } = await import('@detiq/core');
-
     const {
       generateScriptAligned, deriveRepoTargetPath, validateGeneratedScript,
       mapWithConcurrency, SCRIPT_GEN_CONCURRENCY,
@@ -3772,7 +4071,7 @@ function buildPRBody(scripts: any[], projectId: string): string {
 // ── NEW_PAGES_PIPELINE — post-crawl: detect new screens, generate tests + scripts ──
 
 async function processNewPagesPipelineJob(jobId: string, config: any) {
-  const { tenantId, projectId, crawlJobId, repoConnectionId, autoPR = false } = config;
+  const { tenantId, projectId, crawlJobId: _crawlJobId, repoConnectionId, autoPR = false } = config;
   try {
     await updateJobProgress(tenantId, jobId, 5, 'Detecting new screens');