@detiq/crawler 1.1.27 → 1.1.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.27",
+  "version": "1.1.29",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -7013,6 +7013,18 @@ async function runAuthDiscoveryPhase(
       }
 
       const finalUrl = page.url?.() ?? current.url;
+
+      // Skip auth seed paths that redirect to a non-auth URL — the page captured would be the
+      // post-login destination (e.g. dashboard) not the actual auth screen.
+      if (current.source === 'seed_path' && finalUrl !== current.url) {
+        try {
+          const origPath = new URL(current.url).pathname;
+          const finalPath = new URL(finalUrl).pathname;
+          const isAuthSeed = /login|sign-?in|signin|auth|sso|forgot-?password|reset-?password|2fa|mfa/i.test(origPath);
+          if (isAuthSeed && finalPath !== origPath) continue;
+        } catch { /* non-parseable URL — allow capture */ }
+      }
+
       const captureKey = normalizeAuthUrl(finalUrl);
       if (!capturedKeys.has(captureKey)) {
         const result = await captureAuthDiscoveryScreen(page, opts, current.url, current.source, ordinal++);

package/src/job-queue.test.ts

@@ -2,7 +2,7 @@ import crypto from 'node:crypto';
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { prisma, prismaAdmin } from '@detiq/database';
-import { claimNextQueuedJob, claimSpecificJob, requeueStaleRunningJobs } from './job-queue.js';
+import { claimNextQueuedJob, claimSpecificJob, requeueStaleRunningJobs, classifyFailure, FailureType, recordJobFailure, resumeJob } from './job-queue.js';
 
 function id(prefix: string) {
   return `${prefix}_${crypto.randomUUID()}`;
@@ -13,7 +13,7 @@ async function makeFixtureProject() {
   const projectId = id('project');
   await prisma.$transaction(async (tx) => {
     await tx.$executeRawUnsafe(`SELECT set_config('app.current_tenant', $1, true)`, tenantId);
-    await tx.tenant.create({ data: { id: tenantId, name: 'Job Queue Test Tenant', slug: tenantId } });
+    await tx.tenant.create({ data: { id: tenantId, name: 'Job Queue Test Tenant', slug: tenantId, isolation: 'SHARED', region: 'US' } });
     await tx.project.create({ data: { id: projectId, tenantId, name: 'Job Queue Test Project' } });
   });
   return { tenantId, projectId };
@@ -118,3 +118,70 @@ test('requeueStaleRunningJobs: a job with no heartbeat falls back to startedAt',
   const job = await prismaAdmin.platformJob.findUnique({ where: { id: jobId } });
   assert.equal(job!.status, 'QUEUED');
 });
+
+test('classifyFailure: classifies errors accurately across categories', () => {
+  assert.equal(classifyFailure(new Error('ETIMEDOUT: network connection reset')), FailureType.TRANSIENT);
+  assert.equal(classifyFailure(new Error('Rate limit exceeded: 429 Too Many Requests')), FailureType.TRANSIENT);
+  assert.equal(classifyFailure(new Error('FATAL: master_enc_key environment variable missing')), FailureType.FATAL);
+  assert.equal(classifyFailure(new Error('PrismaClientInitializationError: database connection refused')), FailureType.INFRASTRUCTURE);
+  assert.equal(classifyFailure(new Error('Worker exited unexpectedly: sigkill out of memory')), FailureType.INFRASTRUCTURE);
+  assert.equal(classifyFailure(new Error('Validation failed: 400 bad request')), FailureType.LOGICAL);
+});
+
+test('recordJobFailure: handles transient error with RETRY_WAIT and exponential backoff', async () => {
+  const { tenantId, projectId } = await makeFixtureProject();
+  const jobId = await insertQueuedJob(tenantId, projectId);
+
+  await recordJobFailure(tenantId, jobId, new Error('network timeout 503'));
+
+  const job = await prismaAdmin.platformJob.findUnique({ where: { id: jobId } });
+  assert.equal(job!.status, 'RETRY_WAIT');
+  assert.equal(job!.failureType, 'TRANSIENT');
+  assert.equal(job!.retryCount, 1);
+  assert.equal(job!.dlq, false);
+  assert.ok(job!.nextRetryAt && job!.nextRetryAt > new Date());
+});
+
+test('recordJobFailure: immediately routes FATAL errors to DLQ with 0 retries', async () => {
+  const { tenantId, projectId } = await makeFixtureProject();
+  const jobId = await insertQueuedJob(tenantId, projectId);
+
+  await recordJobFailure(tenantId, jobId, new Error('invalid configuration: master_enc_key not set'));
+
+  const job = await prismaAdmin.platformJob.findUnique({ where: { id: jobId } });
+  assert.equal(job!.status, 'FAILED');
+  assert.equal(job!.failureType, 'FATAL');
+  assert.equal(job!.dlq, true);
+  assert.ok(job!.dlqAt);
+  assert.ok(job!.dlqReason?.includes('master_enc_key'));
+});
+
+test('resumeJob: restores visitedUrls and pendingQueue from PlatformJob.checkpoint column', async () => {
+  const { tenantId, projectId } = await makeFixtureProject();
+  const jobId = id('job');
+  await prismaAdmin.platformJob.create({
+    data: {
+      id: jobId,
+      tenantId,
+      projectId,
+      type: 'CRAWL',
+      status: 'FAILED',
+      config: { tenantId, projectId, appUrl: 'https://example.com' },
+      checkpoint: {
+        visitedUrls: ['https://example.com/page1', 'https://example.com/page2'],
+        pendingQueue: ['https://example.com/page3'],
+      },
+      checkpointAt: new Date(),
+    },
+  });
+
+  const resumedJobId = await resumeJob(tenantId, jobId);
+  assert.ok(resumedJobId);
+
+  const resumedJob = await prismaAdmin.platformJob.findUnique({ where: { id: resumedJobId } });
+  assert.ok(resumedJob);
+  const cfg = resumedJob!.config as any;
+  assert.deepEqual(cfg.startUrls, ['https://example.com/page3']);
+  assert.ok(cfg.skipUrls.includes('https://example.com/page1'));
+  assert.ok(cfg.skipUrls.includes('https://example.com/page2'));
+});