@dealcrawl/sdk 2.5.0 → 2.6.0

package/dist/index.js

@@ -12,33 +12,164 @@ var DEFAULT_CONFIG = {
 
 // src/error.ts
 var ERROR_CODES = {
-  // Authentication
-  INVALID_API_KEY: "INVALID_API_KEY",
-  MISSING_API_KEY: "MISSING_API_KEY",
-  API_KEY_EXPIRED: "API_KEY_EXPIRED",
-  ACCOUNT_SUSPENDED: "ACCOUNT_SUSPENDED",
-  // Rate limiting
+  // ============================================
+  // Authentication & Authorization (4xx)
+  // ============================================
+  /** Missing or invalid Authorization header */
+  AUTH_MISSING_HEADER: "AUTH_MISSING_HEADER",
+  /** The provided API key is invalid or expired */
+  AUTH_INVALID_API_KEY: "AUTH_INVALID_API_KEY",
+  /** Your API key has expired */
+  AUTH_API_KEY_EXPIRED: "AUTH_API_KEY_EXPIRED",
+  /** Your API key has been revoked */
+  AUTH_API_KEY_REVOKED: "AUTH_API_KEY_REVOKED",
+  /** API key does not have required permissions */
+  AUTH_INSUFFICIENT_SCOPE: "AUTH_INSUFFICIENT_SCOPE",
+  /** Your account has been suspended */
+  AUTH_ACCOUNT_SUSPENDED: "AUTH_ACCOUNT_SUSPENDED",
+  // Legacy aliases (for backward compatibility)
+  /** @deprecated Use AUTH_INVALID_API_KEY instead */
+  INVALID_API_KEY: "AUTH_INVALID_API_KEY",
+  /** @deprecated Use AUTH_MISSING_HEADER instead */
+  MISSING_API_KEY: "AUTH_MISSING_HEADER",
+  /** @deprecated Use AUTH_API_KEY_EXPIRED instead */
+  API_KEY_EXPIRED: "AUTH_API_KEY_EXPIRED",
+  /** @deprecated Use AUTH_ACCOUNT_SUSPENDED instead */
+  ACCOUNT_SUSPENDED: "AUTH_ACCOUNT_SUSPENDED",
+  // ============================================
+  // Rate Limiting (429)
+  // ============================================
+  /** Rate limit exceeded. Please try again later */
   RATE_LIMIT_EXCEEDED: "RATE_LIMIT_EXCEEDED",
-  QUOTA_EXCEEDED: "QUOTA_EXCEEDED",
-  // Validation
-  INVALID_URL: "INVALID_URL",
-  INVALID_REQUEST: "INVALID_REQUEST",
-  MISSING_REQUIRED_FIELD: "MISSING_REQUIRED_FIELD",
-  // Job errors
-  JOB_NOT_FOUND: "JOB_NOT_FOUND",
-  JOB_FAILED: "JOB_FAILED",
+  /** Monthly quota exceeded. Upgrade your plan for more */
+  RATE_QUOTA_EXCEEDED: "RATE_QUOTA_EXCEEDED",
+  /** Concurrent job limit reached */
+  RATE_CONCURRENT_LIMIT: "RATE_CONCURRENT_LIMIT",
+  // Legacy aliases
+  /** @deprecated Use RATE_QUOTA_EXCEEDED instead */
+  QUOTA_EXCEEDED: "RATE_QUOTA_EXCEEDED",
+  // ============================================
+  // Validation (400, 422)
+  // ============================================
+  /** The provided input is invalid */
+  VALID_INVALID_INPUT: "VALID_INVALID_INPUT",
+  /** Required field is missing */
+  VALID_MISSING_FIELD: "VALID_MISSING_FIELD",
+  /** The provided URL is not valid */
+  VALID_INVALID_URL: "VALID_INVALID_URL",
+  /** Invalid schema format */
+  VALID_INVALID_SCHEMA: "VALID_INVALID_SCHEMA",
+  /** Invalid format */
+  VALID_INVALID_FORMAT: "VALID_INVALID_FORMAT",
+  // Legacy aliases
+  /** @deprecated Use VALID_INVALID_URL instead */
+  INVALID_URL: "VALID_INVALID_URL",
+  /** @deprecated Use VALID_INVALID_INPUT instead */
+  INVALID_REQUEST: "VALID_INVALID_INPUT",
+  /** @deprecated Use VALID_MISSING_FIELD instead */
+  MISSING_REQUIRED_FIELD: "VALID_MISSING_FIELD",
+  // ============================================
+  // Resources (404)
+  // ============================================
+  /** The requested resource was not found */
+  RESOURCE_NOT_FOUND: "RESOURCE_NOT_FOUND",
+  /** Job not found */
+  RESOURCE_JOB_NOT_FOUND: "RESOURCE_JOB_NOT_FOUND",
+  /** Webhook not found */
+  RESOURCE_WEBHOOK_NOT_FOUND: "RESOURCE_WEBHOOK_NOT_FOUND",
+  /** API key not found */
+  RESOURCE_KEY_NOT_FOUND: "RESOURCE_KEY_NOT_FOUND",
+  // Legacy aliases
+  /** @deprecated Use RESOURCE_JOB_NOT_FOUND instead */
+  JOB_NOT_FOUND: "RESOURCE_JOB_NOT_FOUND",
+  // ============================================
+  // Job Processing (422, 503)
+  // ============================================
+  /** Failed to create job */
+  JOB_CREATION_FAILED: "JOB_CREATION_FAILED",
+  /** Job timed out */
   JOB_TIMEOUT: "JOB_TIMEOUT",
-  // Scraping errors
-  FETCH_FAILED: "FETCH_FAILED",
-  PARSE_FAILED: "PARSE_FAILED",
-  BLOCKED_BY_ROBOTS: "BLOCKED_BY_ROBOTS",
-  CAPTCHA_DETECTED: "CAPTCHA_DETECTED",
-  SITE_UNREACHABLE: "SITE_UNREACHABLE",
-  // System errors
+  /** Job processing failed */
+  JOB_FAILED: "JOB_FAILED",
+  /** Job was cancelled */
+  JOB_CANCELLED: "JOB_CANCELLED",
+  // ============================================
+  // Scraping/Crawling (422, 503)
+  // ============================================
+  /** Failed to fetch URL */
+  SCRAPE_FETCH_FAILED: "SCRAPE_FETCH_FAILED",
+  /** Failed to parse page content */
+  SCRAPE_PARSE_FAILED: "SCRAPE_PARSE_FAILED",
+  /** URL is blocked by robots.txt */
+  SCRAPE_BLOCKED_BY_ROBOTS: "SCRAPE_BLOCKED_BY_ROBOTS",
+  /** CAPTCHA detected on page */
+  SCRAPE_CAPTCHA_DETECTED: "SCRAPE_CAPTCHA_DETECTED",
+  /** Site is unreachable */
+  SCRAPE_SITE_UNREACHABLE: "SCRAPE_SITE_UNREACHABLE",
+  /** Scraping operation timed out */
+  SCRAPE_TIMEOUT: "SCRAPE_TIMEOUT",
+  // Legacy aliases
+  /** @deprecated Use SCRAPE_FETCH_FAILED instead */
+  FETCH_FAILED: "SCRAPE_FETCH_FAILED",
+  /** @deprecated Use SCRAPE_PARSE_FAILED instead */
+  PARSE_FAILED: "SCRAPE_PARSE_FAILED",
+  /** @deprecated Use SCRAPE_BLOCKED_BY_ROBOTS instead */
+  BLOCKED_BY_ROBOTS: "SCRAPE_BLOCKED_BY_ROBOTS",
+  /** @deprecated Use SCRAPE_CAPTCHA_DETECTED instead */
+  CAPTCHA_DETECTED: "SCRAPE_CAPTCHA_DETECTED",
+  /** @deprecated Use SCRAPE_SITE_UNREACHABLE instead */
+  SITE_UNREACHABLE: "SCRAPE_SITE_UNREACHABLE",
+  // ============================================
+  // Webhooks (400, 422)
+  // ============================================
+  /** A webhook with this event and URL already exists */
+  WEBHOOK_ALREADY_EXISTS: "WEBHOOK_ALREADY_EXISTS",
+  /** Webhook URL is not valid or not publicly accessible */
+  WEBHOOK_INVALID_URL: "WEBHOOK_INVALID_URL",
+  /** Webhook replay attack detected */
+  WEBHOOK_REPLAY_DETECTED: "WEBHOOK_REPLAY_DETECTED",
+  /** Webhook request is too old */
+  WEBHOOK_EXPIRED: "WEBHOOK_EXPIRED",
+  /** Webhook signature is invalid */
+  WEBHOOK_INVALID_SIGNATURE: "WEBHOOK_INVALID_SIGNATURE",
+  // ============================================
+  // AI Extraction (422)
+  // ============================================
+  /** AI extraction failed to process content */
+  AI_EXTRACTION_FAILED: "AI_EXTRACTION_FAILED",
+  /** Failed to parse JSON from AI response */
+  AI_JSON_PARSE_FAILED: "AI_JSON_PARSE_FAILED",
+  /** AI response did not match expected schema */
+  AI_SCHEMA_VALIDATION_FAILED: "AI_SCHEMA_VALIDATION_FAILED",
+  /** No price information could be extracted */
+  AI_NO_PRICE_FOUND: "AI_NO_PRICE_FOUND",
+  /** Extraction confidence is below acceptable threshold */
+  AI_LOW_CONFIDENCE: "AI_LOW_CONFIDENCE",
+  /** Only partial data could be extracted */
+  AI_PARTIAL_RESULT: "AI_PARTIAL_RESULT",
+  /** Fallback extraction was used */
+  AI_FALLBACK_USED: "AI_FALLBACK_USED",
+  /** All fallback strategies failed */
+  AI_FALLBACK_FAILED: "AI_FALLBACK_FAILED",
+  /** LLM provider rate limit exceeded */
+  AI_PROVIDER_RATE_LIMIT: "AI_PROVIDER_RATE_LIMIT",
+  /** LLM provider authentication failed */
+  AI_PROVIDER_AUTH_FAILED: "AI_PROVIDER_AUTH_FAILED",
+  /** LLM provider timeout */
+  AI_PROVIDER_TIMEOUT: "AI_PROVIDER_TIMEOUT",
+  // ============================================
+  // System Errors (500, 503)
+  // ============================================
+  /** An internal server error occurred */
   INTERNAL_ERROR: "INTERNAL_ERROR",
-  SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE",
+  /** Database operation failed */
+  DATABASE_ERROR: "DATABASE_ERROR",
+  /** Redis operation failed */
   REDIS_ERROR: "REDIS_ERROR",
-  DATABASE_ERROR: "DATABASE_ERROR"
+  /** External service error */
+  EXTERNAL_SERVICE_ERROR: "EXTERNAL_SERVICE_ERROR",
+  /** Service is temporarily unavailable */
+  SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE"
 };
 var DealCrawlError = class _DealCrawlError extends Error {
   /** Error code from ERROR_CODES */
@@ -62,14 +193,19 @@ var DealCrawlError = class _DealCrawlError extends Error {
   }
   /**
    * Check if the error is retryable
-   * Rate limits and transient errors are retryable
+   * Rate limits, transient errors, and AI provider issues are retryable
    */
   isRetryable() {
     const retryableCodes = [
       ERROR_CODES.RATE_LIMIT_EXCEEDED,
+      ERROR_CODES.RATE_CONCURRENT_LIMIT,
       ERROR_CODES.SERVICE_UNAVAILABLE,
       ERROR_CODES.REDIS_ERROR,
-      ERROR_CODES.DATABASE_ERROR
+      ERROR_CODES.DATABASE_ERROR,
+      ERROR_CODES.EXTERNAL_SERVICE_ERROR,
+      ERROR_CODES.AI_PROVIDER_RATE_LIMIT,
+      ERROR_CODES.AI_PROVIDER_TIMEOUT,
+      ERROR_CODES.SCRAPE_TIMEOUT
     ];
     return retryableCodes.includes(this.code);
   }
@@ -77,25 +213,92 @@ var DealCrawlError = class _DealCrawlError extends Error {
    * Check if the error is due to authentication issues
    */
   isAuthError() {
-    const authCodes = [
-      ERROR_CODES.INVALID_API_KEY,
-      ERROR_CODES.MISSING_API_KEY,
-      ERROR_CODES.API_KEY_EXPIRED,
-      ERROR_CODES.ACCOUNT_SUSPENDED
-    ];
-    return authCodes.includes(this.code);
+    return this.code.startsWith("AUTH_");
   }
   /**
    * Check if the error is due to rate limiting
    */
   isRateLimited() {
-    return this.code === ERROR_CODES.RATE_LIMIT_EXCEEDED;
+    return this.code === ERROR_CODES.RATE_LIMIT_EXCEEDED || this.code === ERROR_CODES.RATE_CONCURRENT_LIMIT;
   }
   /**
    * Check if the error is due to quota exceeded
    */
   isQuotaExceeded() {
-    return this.code === ERROR_CODES.QUOTA_EXCEEDED;
+    return this.code === ERROR_CODES.RATE_QUOTA_EXCEEDED;
+  }
+  /**
+   * Check if the error is due to AI extraction issues
+   */
+  isExtractionError() {
+    return this.code.startsWith("AI_");
+  }
+  /**
+   * Check if result is partial (some data extracted but incomplete)
+   */
+  isPartialResult() {
+    return this.code === ERROR_CODES.AI_PARTIAL_RESULT || this.code === ERROR_CODES.AI_LOW_CONFIDENCE || this.code === ERROR_CODES.AI_FALLBACK_USED;
+  }
+  /**
+   * Check if extraction failed due to schema validation
+   */
+  isSchemaValidationError() {
+    return this.code === ERROR_CODES.AI_SCHEMA_VALIDATION_FAILED || this.code === ERROR_CODES.AI_JSON_PARSE_FAILED || this.code === ERROR_CODES.VALID_INVALID_SCHEMA;
+  }
+  /**
+   * Check if the error is due to validation issues
+   */
+  isValidationError() {
+    return this.code.startsWith("VALID_");
+  }
+  /**
+   * Check if the error is due to scraping issues
+   */
+  isScrapeError() {
+    return this.code.startsWith("SCRAPE_");
+  }
+  /**
+   * Check if the error is due to webhook issues
+   */
+  isWebhookError() {
+    return this.code.startsWith("WEBHOOK_");
+  }
+  /**
+   * Check if the error is a server error (5xx)
+   */
+  isServerError() {
+    return this.code === ERROR_CODES.INTERNAL_ERROR || this.code === ERROR_CODES.DATABASE_ERROR || this.code === ERROR_CODES.REDIS_ERROR || this.code === ERROR_CODES.EXTERNAL_SERVICE_ERROR || this.code === ERROR_CODES.SERVICE_UNAVAILABLE;
+  }
+  /**
+   * Check if the error is a client error (4xx)
+   */
+  isClientError() {
+    return this.isAuthError() || this.isValidationError() || this.code.startsWith("RESOURCE_") || this.isWebhookError();
+  }
+  /**
+   * Check if fallback was used during extraction
+   */
+  isFallbackUsed() {
+    return this.code === ERROR_CODES.AI_FALLBACK_USED;
+  }
+  /**
+   * Check if all fallback strategies failed
+   */
+  isFallbackFailed() {
+    return this.code === ERROR_CODES.AI_FALLBACK_FAILED;
+  }
+  /**
+   * Check if error is due to AI provider issues
+   */
+  isAIProviderError() {
+    return this.code === ERROR_CODES.AI_PROVIDER_RATE_LIMIT || this.code === ERROR_CODES.AI_PROVIDER_AUTH_FAILED || this.code === ERROR_CODES.AI_PROVIDER_TIMEOUT;
+  }
+  /**
+   * Get extraction-specific error details
+   */
+  getExtractionDetails() {
+    if (!this.isExtractionError()) return void 0;
+    return this.details;
   }
   /**
    * Convert error to JSON-serializable object
@@ -137,23 +340,92 @@ var DealCrawlError = class _DealCrawlError extends Error {
 function mapStatusCodeToErrorCode(statusCode) {
   switch (statusCode) {
     case 400:
-      return ERROR_CODES.INVALID_REQUEST;
+      return ERROR_CODES.VALID_INVALID_INPUT;
     case 401:
-      return ERROR_CODES.INVALID_API_KEY;
+      return ERROR_CODES.AUTH_INVALID_API_KEY;
     case 403:
-      return ERROR_CODES.ACCOUNT_SUSPENDED;
+      return ERROR_CODES.AUTH_ACCOUNT_SUSPENDED;
     case 404:
-      return ERROR_CODES.JOB_NOT_FOUND;
+      return ERROR_CODES.RESOURCE_NOT_FOUND;
+    case 408:
+      return ERROR_CODES.JOB_TIMEOUT;
+    case 422:
+      return ERROR_CODES.VALID_INVALID_INPUT;
     case 429:
       return ERROR_CODES.RATE_LIMIT_EXCEEDED;
     case 500:
       return ERROR_CODES.INTERNAL_ERROR;
+    case 502:
+      return ERROR_CODES.EXTERNAL_SERVICE_ERROR;
     case 503:
       return ERROR_CODES.SERVICE_UNAVAILABLE;
     default:
       return ERROR_CODES.INTERNAL_ERROR;
   }
 }
+var ERROR_MESSAGES = {
+  // Authentication
+  AUTH_MISSING_HEADER: "Missing or invalid Authorization header",
+  AUTH_INVALID_API_KEY: "The provided API key is invalid or expired",
+  AUTH_API_KEY_EXPIRED: "Your API key has expired",
+  AUTH_API_KEY_REVOKED: "Your API key has been revoked",
+  AUTH_INSUFFICIENT_SCOPE: "API key does not have required permissions",
+  AUTH_ACCOUNT_SUSPENDED: "Your account has been suspended",
+  // Rate Limiting
+  RATE_LIMIT_EXCEEDED: "Rate limit exceeded. Please try again later",
+  RATE_QUOTA_EXCEEDED: "Monthly quota exceeded. Upgrade your plan for more",
+  RATE_CONCURRENT_LIMIT: "Concurrent job limit reached. Wait for existing jobs to complete",
+  // Validation
+  VALID_INVALID_INPUT: "The provided input is invalid",
+  VALID_MISSING_FIELD: "Required field is missing",
+  VALID_INVALID_URL: "The provided URL is not valid",
+  VALID_INVALID_SCHEMA: "Invalid schema format",
+  VALID_INVALID_FORMAT: "Invalid format",
+  // Resources
+  RESOURCE_NOT_FOUND: "The requested resource was not found",
+  RESOURCE_JOB_NOT_FOUND: "Job not found",
+  RESOURCE_WEBHOOK_NOT_FOUND: "Webhook not found",
+  RESOURCE_KEY_NOT_FOUND: "API key not found",
+  // Job Processing
+  JOB_CREATION_FAILED: "Failed to create job",
+  JOB_TIMEOUT: "Job timed out",
+  JOB_FAILED: "Job processing failed",
+  JOB_CANCELLED: "Job was cancelled",
+  // Scraping
+  SCRAPE_FETCH_FAILED: "Failed to fetch URL",
+  SCRAPE_PARSE_FAILED: "Failed to parse page content",
+  SCRAPE_BLOCKED_BY_ROBOTS: "URL is blocked by robots.txt",
+  SCRAPE_CAPTCHA_DETECTED: "CAPTCHA detected on page",
+  SCRAPE_SITE_UNREACHABLE: "Site is unreachable",
+  SCRAPE_TIMEOUT: "Scraping operation timed out",
+  // Webhooks
+  WEBHOOK_ALREADY_EXISTS: "A webhook with this event and URL already exists",
+  WEBHOOK_INVALID_URL: "Webhook URL is not valid or not publicly accessible",
+  WEBHOOK_REPLAY_DETECTED: "Webhook replay attack detected",
+  WEBHOOK_EXPIRED: "Webhook request is too old",
+  WEBHOOK_INVALID_SIGNATURE: "Webhook signature is invalid",
+  // AI Extraction
+  AI_EXTRACTION_FAILED: "AI extraction failed to process content",
+  AI_JSON_PARSE_FAILED: "Failed to parse JSON from AI response",
+  AI_SCHEMA_VALIDATION_FAILED: "AI response did not match expected schema",
+  AI_NO_PRICE_FOUND: "No price information could be extracted",
+  AI_LOW_CONFIDENCE: "Extraction confidence is below acceptable threshold",
+  AI_PARTIAL_RESULT: "Only partial data could be extracted",
+  AI_FALLBACK_USED: "Fallback extraction was used",
+  AI_FALLBACK_FAILED: "All fallback strategies failed",
+  AI_PROVIDER_RATE_LIMIT: "LLM provider rate limit exceeded",
+  AI_PROVIDER_AUTH_FAILED: "LLM provider authentication failed",
+  AI_PROVIDER_TIMEOUT: "LLM provider timeout",
+  // System
+  INTERNAL_ERROR: "An internal server error occurred",
+  DATABASE_ERROR: "Database operation failed",
+  REDIS_ERROR: "Redis operation failed",
+  EXTERNAL_SERVICE_ERROR: "External service error",
+  SERVICE_UNAVAILABLE: "Service is temporarily unavailable"
+};
+function getErrorMessage(code) {
+  return ERROR_MESSAGES[code] || "An unknown error occurred";
+}
 
 // src/utils/request.ts
 function buildQueryString(params) {
@@ -2814,6 +3086,7 @@ exports.DealCrawl = DealCrawl;
 exports.DealCrawlError = DealCrawlError;
 exports.DorkResource = DorkResource;
 exports.ERROR_CODES = ERROR_CODES;
+exports.ERROR_MESSAGES = ERROR_MESSAGES;
 exports.ExtractResource = ExtractResource;
 exports.KeysResource = KeysResource;
 exports.ScrapeResource = ScrapeResource;
@@ -2821,6 +3094,7 @@ exports.SearchResource = SearchResource;
 exports.StatusResource = StatusResource;
 exports.WebhooksResource = WebhooksResource;
 exports.default = DealCrawl;
+exports.getErrorMessage = getErrorMessage;
 exports.pollUntil = pollUntil;
 exports.waitForAll = waitForAll;
 exports.waitForAny = waitForAny;