@detiq/crawler 1.1.23 → 1.1.25

package/src/sqs-queue.ts

@@ -1,21 +1,96 @@
-import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
+import {
+  SQSClient,
+  SendMessageCommand,
+  ReceiveMessageCommand,
+  DeleteMessageCommand,
+  GetQueueAttributesCommand,
+} from '@aws-sdk/client-sqs';
 
 const sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });
 
-export async function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise<void> {
-  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');
+function getQueueUrl(): string {
+  const url = process.env.SQS_QUEUE_URL ?? process.env.SQS_CRAWL_QUEUE_URL;
+  if (!url) throw new Error('SQS_QUEUE_URL not configured');
+  return url;
+}
+
+function getDlqUrl(): string | null {
+  return process.env.SQS_DLQ_URL ?? null;
+}
+
+export async function sendJobToSqs(jobId: string): Promise<void> {
   await sqs.send(new SendMessageCommand({
-    QueueUrl: queueUrl,
-    MessageBody: JSON.stringify({ jobId, tenantId, config }),
+    QueueUrl: getQueueUrl(),
+    MessageBody: JSON.stringify({ jobId }),
   }));
 }
 
-export async function startSqsWorker(): Promise<void> {
-  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');
+/** Returns the approximate number of messages in the SQS DLQ. */
+export async function getDlqStats(): Promise<{ approximateMessageCount: number; dlqUrl: string | null }> {
+  const dlqUrl = getDlqUrl();
+  if (!dlqUrl) return { approximateMessageCount: 0, dlqUrl: null };
+  try {
+    const { Attributes = {} } = await sqs.send(new GetQueueAttributesCommand({
+      QueueUrl: dlqUrl,
+      AttributeNames: ['ApproximateNumberOfMessages'],
+    }));
+    return {
+      approximateMessageCount: parseInt(Attributes.ApproximateNumberOfMessages ?? '0', 10),
+      dlqUrl,
+    };
+  } catch (err: any) {
+    console.error('[sqs-dlq] getDlqStats error:', err.message);
+    return { approximateMessageCount: 0, dlqUrl };
+  }
+}
+
+/**
+ * Redrive up to `maxMessages` messages from the SQS DLQ back to the main queue.
+ * Each message is received from the DLQ, re-sent to the main queue, then deleted
+ * from the DLQ. Returns the number of messages successfully redriven.
+ */
+export async function redriveFromDlq(maxMessages = 10): Promise<number> {
+  const dlqUrl = getDlqUrl();
+  if (!dlqUrl) {
+    console.warn('[sqs-dlq] SQS_DLQ_URL not configured — cannot redrive');
+    return 0;
+  }
+  const mainUrl = getQueueUrl();
+  let totalRedriven = 0;
+
+  while (totalRedriven < maxMessages) {
+    const batch = Math.min(10, maxMessages - totalRedriven);
+    const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+      QueueUrl: dlqUrl,
+      MaxNumberOfMessages: batch,
+      WaitTimeSeconds: 1,
+    }));
+    if (Messages.length === 0) break;
+
+    for (const msg of Messages) {
+      try {
+        await sqs.send(new SendMessageCommand({
+          QueueUrl: mainUrl,
+          MessageBody: msg.Body!,
+        }));
+        await sqs.send(new DeleteMessageCommand({
+          QueueUrl: dlqUrl,
+          ReceiptHandle: msg.ReceiptHandle!,
+        }));
+        totalRedriven++;
+        console.log('[sqs-dlq] redriven message:', msg.MessageId);
+      } catch (err: any) {
+        console.error('[sqs-dlq] redrive error for message', msg.MessageId, ':', err.message);
+      }
+    }
+  }
+
+  return totalRedriven;
+}
 
-  const { processJob } = await import('./job-queue.js');
+export async function startSqsWorker(): Promise<void> {
+  const queueUrl = getQueueUrl();
+  const { claimSpecificJob, dispatchClaimedJob } = await import('./job-queue.js');
   console.log('[sqs-worker] starting, queue:', queueUrl);
 
   while (true) {
@@ -31,14 +106,19 @@ export async function startSqsWorker(): Promise<void> {
         try {
           const data = JSON.parse(msg.Body!);
           jobId = data.jobId;
-          await processJob(data.jobId, data.config);
+          // Atomically move QUEUED → RUNNING; returns null if another worker beat us.
+          const claimed = await claimSpecificJob(jobId);
+          if (claimed) {
+            await dispatchClaimedJob(claimed);
+          }
+          // Delete whether we ran it or another worker did — message is handled either way.
           await sqs.send(new DeleteMessageCommand({
             QueueUrl: queueUrl,
             ReceiptHandle: msg.ReceiptHandle!,
           }));
         } catch (err: any) {
           console.error(`[sqs-worker] job ${jobId} failed — leaving for SQS redrive:`, err.message);
-          // Do NOT delete — SQS redrive policy + DLQ handles retry
+          // Do NOT delete — SQS visibility timeout expires and DLQ handles retry.
         }
       }));
     } catch (err: any) {

package/src/worker.ts

@@ -14,8 +14,19 @@ import { logger } from './logger.js';
 async function main() {
   logger.info('[worker] starting standalone crawler worker process');
 
-  // Durable DB-backed queue — claims every non-CRAWL job type, and CRAWL too when
-  // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.
+  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
+    // SQS mode: messages arrive via long-poll; claimSpecificJob atomically moves
+    // QUEUED→RUNNING before dispatch, so multiple worker replicas are safe.
+    // DB poll loop also runs as a safety net for jobs whose SQS message was lost.
+    const { startSqsWorker } = await import('./sqs-queue.js');
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, '[worker] SQS worker crashed');
+      process.exit(1);
+    });
+  }
+
+  // DB-backed poll loop — primary in non-SQS mode, safety net in SQS mode.
+  // Claims all job types. Safe to run in many worker replicas (FOR UPDATE SKIP LOCKED).
   startJobWorkerLoop();
 
   // Daily data-retention purge: runs immediately on startup then every 24 h.
@@ -28,15 +39,6 @@ async function main() {
   };
   runPurge();
   setInterval(runPurge, 24 * 60 * 60 * 1000);
-
-  // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.
-  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
-    const { startSqsWorker } = await import('./sqs-queue.js');
-    startSqsWorker().catch((err) => {
-      logger.error({ err: String(err) }, '[worker] SQS worker crashed');
-      process.exit(1);
-    });
-  }
 }
 
 process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });

package/types/crawler.d.ts

@@ -21,6 +21,12 @@ interface PerformanceMetric {
     ttfb?: number;
     cls?: number;
 }
+export interface DiscoveredLinkMeta {
+    url: string;
+    text?: string;
+    landmark?: string;
+    source: 'DOM' | 'AI' | 'INTERACTIVE';
+}
 /**
  * Lightweight pre-crawl link discovery: loads the app URL in a headless (proxy-aware) browser
  * and returns same-site `<a href>` links. Complements sitemap discovery for SPA/app sites that
@@ -60,6 +66,13 @@ export declare function crawlProject(opts: {
     startUrls?: string[];
     /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */
     selectiveUrls?: string[];
+    /**
+     * URL path patterns that bypass the CUID/UUID generated-ID filter during BFS link discovery.
+     * Use {id} as a placeholder for any generated path segment (CUID, UUID, numeric ID, etc.).
+     * Example: ['/projects/{id}/tests', '/workspaces/{id}/settings']
+     * Without this, all URLs containing generated IDs are silently skipped by the BFS.
+     */
+    seedPathPatterns?: string[];
     /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */
     interactive?: boolean;
     /** Optional abort signal — abort() cancels the crawl between page navigations. */
@@ -143,10 +156,17 @@ export declare function crawlProject(opts: {
     autoThrottle?: boolean;
     /** Regex pattern — only crawl URLs whose path+query match this pattern. */
     urlFilter?: string;
+    /** Glob patterns — only enqueue URLs whose pathname matches at least one (supports * and **). */
+    includePatterns?: string[];
+    /** Glob patterns — skip URLs whose pathname matches any of these (supports * and **). */
+    excludePatterns?: string[];
     /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */
     skipUrls?: string[];
     /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */
     knownDomHashes?: string[];
+    /** When false, disables the puppeteer-extra stealth plugin (playwright-extra) for this crawl.
+     *  Default true — stealth is applied unless explicitly set to false. */
+    stealth?: boolean;
     /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
     onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
     /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */

package/types/extractors/branding-extractor.d.ts

@@ -0,0 +1,46 @@
+import type { Page } from 'playwright';
+export interface BrandingColors {
+    primary?: string;
+    accent?: string;
+    background?: string;
+    textPrimary?: string;
+    link?: string;
+}
+export interface BrandingFont {
+    family: string;
+    role: 'heading' | 'body';
+}
+export interface BrandingImages {
+    logo?: string;
+    favicon?: string;
+    ogImage?: string;
+}
+export interface BrandingPersonality {
+    tone?: string;
+    energy?: 'high' | 'medium' | 'low';
+    targetAudience?: string;
+}
+export interface BrandingResult {
+    colorScheme: 'light' | 'dark';
+    colors: BrandingColors;
+    fonts: BrandingFont[];
+    images: BrandingImages;
+    personality?: BrandingPersonality;
+    designSystem?: {
+        framework?: string;
+        componentLibrary?: string;
+    };
+    metadata?: {
+        title?: string;
+        description?: string;
+        themeColor?: string;
+        language?: string;
+    };
+    confidence: {
+        colors: number;
+        fonts: number;
+        images: number;
+        overall: number;
+    };
+}
+export declare function extractBranding(page: Page): Promise<BrandingResult>;

package/types/extractors/summary-extractor.d.ts

@@ -0,0 +1,10 @@
+export interface SummaryResult {
+    summary: string;
+    wordCount: number;
+    method: 'llm' | 'truncated';
+    model?: string;
+}
+export declare function extractSummary(markdown: string, options?: {
+    model?: string;
+    maxTokens?: number;
+}): Promise<SummaryResult>;

package/types/frontier.d.ts

@@ -9,12 +9,20 @@
  * array it replaces, so BFS (the default) has zero behavior change.
  */
 export type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';
+/**
+ * Simple glob-like pattern matching against a URL's pathname.
+ * Supports `*` (matches any single path segment characters, not `/`)
+ * and `**` (matches anything including `/`).
+ */
+export declare function matchesPattern(url: string, patterns: string[]): boolean;
 export declare class CrawlFrontier {
     private items;
     private readonly strategy;
     private readonly keywords;
     private readonly urlFilter;
-    constructor(strategy: TraversalStrategy, seedUrls: string[], keywords?: string[], urlFilter?: string);
+    private readonly includePatterns;
+    private readonly excludePatterns;
+    constructor(strategy: TraversalStrategy, seedUrls: string[], keywords?: string[], urlFilter?: string, includePatterns?: string[], excludePatterns?: string[]);
     get length(): number;
     /**
      * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive

package/types/incremental-diff.d.ts

@@ -57,3 +57,60 @@ export declare function partitionByChange(urls: string[], priorHashes: Map<strin
     toProcess: string[];
     toSkip: string[];
 };
+/** A single line-level change in text content (markdown, html, summary). */
+export interface TextChange {
+    type: 'added' | 'removed';
+    line: number;
+    content: string;
+}
+/** A change to a specific JSON path between two scraped JSON objects. */
+export interface JsonChange {
+    path: string;
+    old: unknown;
+    new: unknown;
+}
+/** Set-based change for collections like links or images. */
+export interface SetChange {
+    added: string[];
+    removed: string[];
+}
+/** Screenshot pixel-level change. changed is percentage (0-100). */
+export interface ImageChange {
+    size: number;
+    changed: number;
+    mask?: string;
+}
+/** All possible diffs for a single monitor tick. */
+export interface MonitorDiffs {
+    markdown?: TextChange[];
+    html?: TextChange[];
+    json?: JsonChange[];
+    summary?: TextChange[];
+    links?: SetChange;
+    images?: SetChange;
+    branding?: JsonChange[];
+    screenshot?: ImageChange;
+}
+/**
+ * Compute line-level text diffs between two strings.
+ * Returns TextChange[] showing exactly which lines were added or removed.
+ * No external dependencies — pure JS Myers-inspired algorithm.
+ */
+export declare function computeTextDiffs(previous: string, current: string): TextChange[];
+/**
+ * Compute set-based diffs for URL collections (links, images).
+ */
+export declare function computeSetDiffs(previous: string[], current: string[]): SetChange;
+/**
+ * Flatten a JSON object into dot-notation path/value pairs.
+ * Used to compute JsonChange[] between two JSON extraction results.
+ */
+export declare function flattenJson(obj: unknown, prefix?: string): Record<string, unknown>;
+/**
+ * Compute JSON path-level diffs between two extraction results.
+ */
+export declare function computeJsonDiffs(previous: unknown, current: unknown): JsonChange[];
+/**
+ * Count total number of changes across all diff types.
+ */
+export declare function countMonitorDiffs(diffs: MonitorDiffs): number;

package/types/index.d.ts

@@ -3,7 +3,7 @@ export * from './visual-diff-verdict.js';
 export * from './auth.js';
 export * from './screen-reader.js';
 export * from './job-queue.js';
-export { startSqsWorker } from './sqs-queue.js';
+export { startSqsWorker, getDlqStats, redriveFromDlq } from './sqs-queue.js';
 export * from './session-capture.js';
 export * from './session-export.js';
 export * from './test-executor.js';

package/types/interactive-probe.d.ts

@@ -37,11 +37,15 @@ export interface ModalObservation {
     title: string;
     content: string;
     fields: string[];
+    stateId?: string;
+    domSnapshot?: string;
 }
 export interface ConditionalReveal {
     triggerElement: string;
     triggerValue: string;
     revealedText: string;
+    stateId?: string;
+    domSnapshot?: string;
 }
 export interface TooltipCapture {
     triggerLabel: string;
@@ -117,7 +121,7 @@ export declare function probeAllScreens(page: any, tenantId: string, projectId:
     url: string;
     screenId?: string;
     elements?: any[];
-}>, onProgress?: (pct: number, step: string) => Promise<void>): Promise<void>;
+}>, onProgress?: (pct: number, step: string) => Promise<void>, jobId?: string): Promise<void>;
 export interface FocusableElement {
     tag: string;
     role: string | null;

package/types/job-queue.d.ts

@@ -67,6 +67,52 @@ export interface PlatformJobConfig {
     ssoScope?: string;
     /** HTTP/HTTPS/SOCKS5 proxy URL — routes browser through this proxy for bot-protected sites. */
     proxyUrl?: string;
+    /** HTML extraction mode. 'normal'=raw, 'reader'=Readability article, 'prune'=strip nav/ads. Default: 'normal'. */
+    htmlMode?: 'normal' | 'reader' | 'prune';
+    /** Per-request screenshot dimensions and quality. Defaults: 1440×900, quality 80. */
+    screenshotConfig?: {
+        fullPage?: boolean;
+        width?: number;
+        height?: number;
+        quality?: number;
+    };
+    /** Enable/disable Playwright stealth plugin. Default: true (existing behaviour). */
+    stealth?: boolean;
+    /** Mock mode — intercept all navigation and return synthetic content. Good for CI. */
+    mock?: boolean | {
+        minKb?: number;
+        maxKb?: number;
+        minSleep?: number;
+        maxSleep?: number;
+    };
+    /** MIME type allowlist — crawler skips URLs returning disallowed content types. */
+    allowedTypes?: string[];
+    /** Glob-style include patterns for URL paths (e.g. '/blog/*'). Only matching paths crawled. */
+    includePatterns?: string[];
+    /** Glob-style exclude patterns for URL paths (e.g. '/admin/*'). Matching paths skipped. */
+    excludePatterns?: string[];
+    /** Allow crawler to follow links to external domains. Default: false. */
+    allowExternal?: boolean;
+    /** Requested output formats for a single-pass multi-format extraction. */
+    requestedFormats?: Array<'markdown' | 'html' | 'links' | 'images' | 'summary' | 'json' | 'branding' | 'screenshot'>;
+    /** LLM chunker configuration for AI extraction passes. */
+    chunkerConfig?: {
+        size?: number | 'dynamic';
+        overlap?: number;
+    };
+    /** ISO-3166-1 alpha-2 country code for geo-targeted proxy routing (e.g. 'us', 'gb'). */
+    proxyCountry?: string;
+    /** When true, after the primary crawl auto-enqueue one CRAWL job per credentialProfile. */
+    crawlAllRoles?: boolean;
+    /** Decrypted credential profiles used when crawlAllRoles is true. */
+    credentialProfiles?: Array<{
+        id: string;
+        name: string;
+        username: string;
+        password: string;
+    }>;
+    /** Set on role-crawl sub-jobs — profile name tagged on discovered screens via roleAccess column. */
+    roleProfileName?: string;
 }
 type ClaimedJob = {
     id: string;
@@ -95,6 +141,13 @@ export declare function requeueStaleRunningJobs(): Promise<void>;
  * Safe to call from multiple replicas simultaneously (each uses its own WORKER_ID).
  */
 export declare function requeueOrphanedJobs(): Promise<void>;
+/** Dispatches a claimed job to its type's processor. Last-resort safety net — every
+ *  process*Job function already catches and records its own failures. */
+export declare function dispatchClaimedJob(job: ClaimedJob): Promise<void>;
+/** Re-promotes FAILED jobs to QUEUED if they are eligible for retry based on per-type limits. */
+export declare function retryEligibleFailedJobs(): Promise<void>;
+/** Promotes FAILED jobs to DLQ if they exceed their per-type retry limits. */
+export declare function promoteToDeadLetterQueue(): Promise<void>;
 /**
  * Starts the durable job worker poll loop. This is what makes job processing
  * survive a restart — enqueueJob()'s immediate same-process claim attempt is

package/types/sqs-queue.d.ts

@@ -1,2 +1,13 @@
-export declare function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise<void>;
+export declare function sendJobToSqs(jobId: string): Promise<void>;
+/** Returns the approximate number of messages in the SQS DLQ. */
+export declare function getDlqStats(): Promise<{
+    approximateMessageCount: number;
+    dlqUrl: string | null;
+}>;
+/**
+ * Redrive up to `maxMessages` messages from the SQS DLQ back to the main queue.
+ * Each message is received from the DLQ, re-sent to the main queue, then deleted
+ * from the DLQ. Returns the number of messages successfully redriven.
+ */
+export declare function redriveFromDlq(maxMessages?: number): Promise<number>;
 export declare function startSqsWorker(): Promise<void>;