@detiq/crawler 0.1.3

package/dist/job-queue.d.ts

@@ -0,0 +1,171 @@
+/**
+ * ZeTa-AI Background Job Queue
+ *
+ * Postgres-backed durable queue. Each job becomes a PlatformJob record.
+ * enqueueJob() writes it QUEUED and, for same-process responsiveness, kicks
+ * an immediate best-effort claim attempt — but durability comes from
+ * startJobWorkerLoop()'s poll: it atomically claims QUEUED jobs (Postgres
+ * `FOR UPDATE SKIP LOCKED`, safe across any number of concurrent worker
+ * processes/containers) and requeues jobs whose heartbeat went stale,
+ * so a crashed/restarted worker never silently loses a job.
+ *
+ * CRAWL-type jobs additionally support AWS SQS as an alternate transport
+ * (JOB_QUEUE_BACKEND=sqs, see sqs-queue.ts) for horizontally-scaled crawler
+ * fleets; every other job type always goes through the DB-backed queue.
+ */
+export declare function generateTOTP(secret: string): string;
+export interface PlatformJobConfig {
+    tenantId: string;
+    projectId: string;
+    appUrl?: string;
+    credentials?: {
+        username: string;
+        password: string;
+        oneTimeCode?: string;
+    };
+    /** Decrypted Playwright storageState JSON — if present, session is restored without AI login. */
+    storageState?: string;
+    screenshotDir?: string;
+    maxScreens?: number;
+    /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
+    startUrls?: string[];
+    /** When provided, crawl ONLY these exact URLs (no link-following). Used for selected-screen recrawl. */
+    selectiveUrls?: string[];
+    /** Device profiles for multi-viewport screenshot pass after main DESKTOP crawl. */
+    deviceProfiles?: string[];
+    /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+    /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
+    authToken?: string;
+    /** Header name for authToken (default: 'Authorization'). */
+    authTokenHeader?: string;
+    /** Prefix for authToken value (default: 'Bearer '). */
+    authTokenPrefix?: string;
+    /** CAPTCHA solving API key (2captcha or CapSolver). */
+    captchaSolverApiKey?: string;
+    /** CAPTCHA solving service provider (default: '2captcha'). */
+    captchaSolverProvider?: '2captcha' | 'capsolver';
+    /**
+     * Path to a Chrome user data directory for profile inheritance (local deployments only).
+     * When set, the crawler exports the existing session from this profile and injects it.
+     */
+    chromeProfilePath?: string;
+    /** MailSlurp API key for email OTP auto-retrieval. */
+    mailslurpApiKey?: string;
+    /** MailSlurp inbox ID to poll for OTP emails. */
+    mailslurpInboxId?: string;
+    /** SSO provider ('okta' | 'azure_ad' | 'generic_oidc'). */
+    ssoProvider?: string;
+    /** SSO domain or tenant ID. */
+    ssoDomain?: string;
+    /** SSO OAuth2 client ID. */
+    ssoClientId?: string;
+    /** SSO OAuth2 client secret (decrypted). */
+    ssoClientSecret?: string;
+    /** SSO scopes (space-separated). Default: 'openid profile'. */
+    ssoScope?: string;
+    /** HTTP/HTTPS/SOCKS5 proxy URL — routes browser through this proxy for bot-protected sites. */
+    proxyUrl?: string;
+}
+type ClaimedJob = {
+    id: string;
+    tenantId: string;
+    projectId: string;
+    type: string;
+    config: any;
+};
+/** Atomically claims one specific QUEUED job for this worker. Returns null if it was already claimed. */
+export declare function claimSpecificJob(jobId: string): Promise<ClaimedJob | null>;
+/** Atomically claims the oldest QUEUED job across all tenants. SKIP LOCKED makes this
+ *  race-safe against any number of other pollers doing the same thing concurrently. */
+export declare function claimNextQueuedJob(): Promise<ClaimedJob | null>;
+/** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
+ *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
+ *  safe to run from every worker process; only affects rows past the timeout. */
+export declare function requeueStaleRunningJobs(): Promise<void>;
+/**
+ * Starts the durable job worker poll loop. This is what makes job processing
+ * survive a restart — enqueueJob()'s immediate same-process claim attempt is
+ * only a latency optimization; if the process dies before that fires (or
+ * mid-job), this loop's next tick claims/requeues it. Call once at process
+ * startup (API server and/or a dedicated crawler worker process — safe to
+ * run in both, and safe to run in many container replicas at once).
+ */
+export declare function startJobWorkerLoop(opts?: {
+    pollIntervalMs?: number;
+    concurrency?: number;
+}): void;
+/** Stops the poll loop. Mainly for tests. */
+export declare function stopJobWorkerLoop(): void;
+/** Create a job record that tracks a synchronous operation (telemetry only, no background processor). */
+export declare function createTrackedJob(type: string, config: {
+    tenantId: string;
+    projectId: string;
+    [k: string]: any;
+}): Promise<string>;
+/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors. */
+export declare function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void>;
+/** Enqueue a generic platform job — returns the job ID immediately. */
+export declare function enqueueJob(type: string, config: any): Promise<string>;
+/** Get the current state of a platform job. */
+export declare function getJob(tenantId: string, jobId: string): Promise<{
+    progress: number;
+    error: string | null;
+    type: string;
+    status: string;
+    tenantId: string;
+    projectId: string;
+    name: string | null;
+    result: import("@prisma/client/runtime/library").JsonValue | null;
+    id: string;
+    config: import("@prisma/client/runtime/library").JsonValue;
+    queuedAt: Date;
+    startedAt: Date | null;
+    finishedAt: Date | null;
+    workerId: string | null;
+    heartbeatAt: Date | null;
+    totalPromptTokens: number | null;
+    totalCompletionTokens: number | null;
+    totalCacheReadTokens: number | null;
+    totalEstimatedCostUsd: number | null;
+    llmCallCount: number;
+} | null>;
+/** List recent jobs for a project. */
+export declare function listJobs(tenantId: string, projectId: string): Promise<{
+    progress: number;
+    error: string | null;
+    type: string;
+    status: string;
+    tenantId: string;
+    projectId: string;
+    name: string | null;
+    result: import("@prisma/client/runtime/library").JsonValue | null;
+    id: string;
+    config: import("@prisma/client/runtime/library").JsonValue;
+    queuedAt: Date;
+    startedAt: Date | null;
+    finishedAt: Date | null;
+    workerId: string | null;
+    heartbeatAt: Date | null;
+    totalPromptTokens: number | null;
+    totalCompletionTokens: number | null;
+    totalCacheReadTokens: number | null;
+    totalEstimatedCostUsd: number | null;
+    llmCallCount: number;
+}[]>;
+/** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
+export declare function cancelJob(tenantId: string, jobId: string): Promise<void>;
+/** Re-enqueue a job using the same config as a previous job. */
+export declare function rerunJob(tenantId: string, jobId: string): Promise<string>;
+export declare function resumeJob(tenantId: string, jobId: string): Promise<string>;
+/** Update progress percentage and current step label on a running job (non-fatal). */
+export declare function updateJobProgress(tenantId: string, jobId: string, progress: number, step: string): Promise<void>;
+export interface ProcessJobOpts {
+    onProgress?: (pct: number) => Promise<void>;
+}
+export declare function processJob(jobId: string, config: any, opts?: ProcessJobOpts): Promise<void>;
+/** After a job finishes, sum LLMCallLog entries for that project during the job window. */
+export declare function aggregateJobTokens(tenantId: string, jobId: string, projectId: string, startedAt: Date): Promise<void>;
+export declare function processDataRetentionPurge(): Promise<void>;
+export {};