@detiq/crawler 1.1.25 → 1.1.27

package/types/crawler.d.ts

@@ -167,6 +167,10 @@ export declare function crawlProject(opts: {
     /** When false, disables the puppeteer-extra stealth plugin (playwright-extra) for this crawl.
      *  Default true — stealth is applied unless explicitly set to false. */
     stealth?: boolean;
+    /** Desktop viewport width in pixels (default: 1440). */
+    viewportWidth?: number;
+    /** Desktop viewport height in pixels (default: 900). */
+    viewportHeight?: number;
     /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
     onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
     /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */

package/types/job-queue.d.ts

@@ -13,6 +13,13 @@
  * (JOB_QUEUE_BACKEND=sqs, see sqs-queue.ts) for horizontally-scaled crawler
  * fleets; every other job type always goes through the DB-backed queue.
  */
+export declare enum FailureType {
+    TRANSIENT = "TRANSIENT",// network timeout, rate limit, 429, 503
+    INFRASTRUCTURE = "INFRASTRUCTURE",// DB connection, OOM, worker crash
+    LOGICAL = "LOGICAL",// bad input, business rule violation, 400/422
+    FATAL = "FATAL"
+}
+export declare function classifyFailure(error: string | Error | unknown): FailureType;
 export declare function generateTOTP(secret: string): string;
 export interface PlatformJobConfig {
     tenantId: string;
@@ -114,6 +121,7 @@ export interface PlatformJobConfig {
     /** Set on role-crawl sub-jobs — profile name tagged on discovered screens via roleAccess column. */
     roleProfileName?: string;
 }
+export declare function getActiveExecutionCount(): number;
 type ClaimedJob = {
     id: string;
     tenantId: string;
@@ -144,7 +152,9 @@ export declare function requeueOrphanedJobs(): Promise<void>;
 /** Dispatches a claimed job to its type's processor. Last-resort safety net — every
  *  process*Job function already catches and records its own failures. */
 export declare function dispatchClaimedJob(job: ClaimedJob): Promise<void>;
-/** Re-promotes FAILED jobs to QUEUED if they are eligible for retry based on per-type limits. */
+/** Re-promotes FAILED jobs to RETRY_WAIT if they are eligible for retry based on per-type limits.
+ *  RETRY_WAIT makes the waiting period visible in the UI. A separate sweep then promotes
+ *  RETRY_WAIT → QUEUED once nextRetryAt <= NOW(). */
 export declare function retryEligibleFailedJobs(): Promise<void>;
 /** Promotes FAILED jobs to DLQ if they exceed their per-type retry limits. */
 export declare function promoteToDeadLetterQueue(): Promise<void>;
@@ -169,7 +179,7 @@ export declare function createTrackedJob(type: string, config: {
     [k: string]: any;
 }): Promise<string>;
 /** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
- * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
+ * GAP-14/15: Uses failure-type classification for per-type backoff on FAILED. */
 export declare function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void>;
 /** Enqueue a generic platform job — returns the job ID immediately. */
 export declare function enqueueJob(type: string, config: any): Promise<string>;
@@ -184,8 +194,10 @@ export declare function getJob(tenantId: string, jobId: string): Promise<{
     status: string;
     tenantId: string;
     projectId: string;
+    inline: boolean;
     config: import("@prisma/client/runtime/library").JsonValue;
     retryCount: number;
+    failureType: string | null;
     queuedAt: Date;
     startedAt: Date | null;
     finishedAt: Date | null;
@@ -199,6 +211,18 @@ export declare function getJob(tenantId: string, jobId: string): Promise<{
     nextRetryAt: Date | null;
     dlq: boolean;
     dlqReason: string | null;
+    leaseExpiresAt: Date | null;
+    fencingToken: string | null;
+    checkpoint: import("@prisma/client/runtime/library").JsonValue | null;
+    checkpointAt: Date | null;
+    parentJobId: string | null;
+    rootJobId: string | null;
+    currentStep: string | null;
+    requeuedAt: Date | null;
+    dlqAt: Date | null;
+    dlqRedriveCount: number;
+    submittedBy: string | null;
+    priority: number;
 } | null>;
 /** List recent jobs for a project. */
 export declare function listJobs(tenantId: string, projectId: string): Promise<{
@@ -211,8 +235,10 @@ export declare function listJobs(tenantId: string, projectId: string): Promise<{
     status: string;
     tenantId: string;
     projectId: string;
+    inline: boolean;
     config: import("@prisma/client/runtime/library").JsonValue;
     retryCount: number;
+    failureType: string | null;
     queuedAt: Date;
     startedAt: Date | null;
     finishedAt: Date | null;
@@ -226,8 +252,23 @@ export declare function listJobs(tenantId: string, projectId: string): Promise<{
     nextRetryAt: Date | null;
     dlq: boolean;
     dlqReason: string | null;
+    leaseExpiresAt: Date | null;
+    fencingToken: string | null;
+    checkpoint: import("@prisma/client/runtime/library").JsonValue | null;
+    checkpointAt: Date | null;
+    parentJobId: string | null;
+    rootJobId: string | null;
+    currentStep: string | null;
+    requeuedAt: Date | null;
+    dlqAt: Date | null;
+    dlqRedriveCount: number;
+    submittedBy: string | null;
+    priority: number;
 }[]>;
-/** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
+/** Cancel a RUNNING or QUEUED job.
+ *  Sets status to CANCEL_REQUESTED (GAP-08) so the worker's abort handler can
+ *  detect the request and set CANCELLED + finishedAt after the underlying work stops.
+ *  The in-process AbortController is also triggered for same-process jobs. */
 export declare function cancelJob(tenantId: string, jobId: string): Promise<void>;
 /** Re-enqueue a job using the same config as a previous job. */
 export declare function rerunJob(tenantId: string, jobId: string): Promise<string>;