@dealcrawl/sdk 2.10.0 → 2.11.0

package/dist/index.js

@@ -1122,6 +1122,245 @@ var AgentResource = class {
   }
 };
 
+// src/resources/auth.ts
+var AuthResource = class {
+  constructor(ctx) {
+    this.ctx = ctx;
+  }
+  /**
+   * Generate SSE authentication token
+   *
+   * Required for browser-based SSE connections because EventSource API
+   * doesn't support custom headers. Token is short-lived (5 minutes).
+   *
+   * Security:
+   * - Requires valid API key (Bearer token)
+   * - Token expires in 5 minutes
+   * - Token can be restricted to specific job
+   * - Token stored in Redis (revocable)
+   *
+   * @example
+   * ```ts
+   * // 1. Generate token
+   * const { token, expiresAt } = await client.auth.generateSSEToken();
+   * console.log(`Token expires at: ${expiresAt}`);
+   *
+   * // 2. Use in browser EventSource
+   * const eventSource = new EventSource(`/v1/events?token=${token}`);
+   *
+   * eventSource.addEventListener('job.completed', (event) => {
+   *   const data = JSON.parse(event.data);
+   *   console.log('Job completed:', data);
+   * });
+   *
+   * // 3. For specific job only
+   * const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
+   * const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
+   * ```
+   */
+  async generateSSEToken(options) {
+    const result = await post(
+      this.ctx,
+      "/v1/auth/sse-token",
+      options ?? {}
+    );
+    return result.data;
+  }
+  /**
+   * Get SSE connection limits for current tier
+   *
+   * Shows how many concurrent SSE connections are allowed
+   * and how many are currently active.
+   *
+   * Tier limits:
+   * - Free: 2 concurrent connections
+   * - Pro: 10 concurrent connections
+   * - Enterprise: 50 concurrent connections
+   *
+   * @example
+   * ```ts
+   * const limits = await client.auth.getLimits();
+   *
+   * console.log(`Tier: ${limits.tier}`);
+   * console.log(`Max connections: ${limits.sse.maxConnections}`);
+   * console.log(`Current connections: ${limits.sse.currentConnections}`);
+   * console.log(`Available: ${limits.sse.available}`);
+   *
+   * // Check before opening new connection
+   * if (limits.sse.available > 0) {
+   *   const token = await client.auth.generateSSEToken();
+   *   const eventSource = new EventSource(`/v1/events?token=${token.token}`);
+   * } else {
+   *   console.error('No available SSE connection slots');
+   * }
+   * ```
+   */
+  async getLimits() {
+    const result = await get(
+      this.ctx,
+      "/v1/auth/limits"
+    );
+    return result.data;
+  }
+};
+
+// src/resources/convert.ts
+var ConvertResource = class {
+  constructor(ctx) {
+    this.ctx = ctx;
+  }
+  /**
+   * Convert HTML to Markdown
+   *
+   * Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
+   * Useful for:
+   * - Converting scraped HTML to markdown for LLM processing
+   * - Cleaning up messy HTML from web pages
+   * - Extracting main content while removing noise (ads, nav, footer)
+   * - Creating documentation from HTML sources
+   *
+   * Features:
+   * - GFM table, strikethrough, and task list support
+   * - Automatic noise removal (scripts, ads, navigation)
+   * - Relative URL resolution
+   * - Custom element exclusion via CSS selectors
+   * - Output length limiting
+   *
+   * @param options - Conversion options
+   * @returns Conversion result with markdown, metadata, and warnings
+   *
+   * @example Basic usage
+   * ```ts
+   * const result = await client.convert.htmlToMarkdown({
+   *   html: "<h1>Product</h1><p>Price: $99</p>"
+   * });
+   * console.log(result.data.markdown);
+   * ```
+   *
+   * @example With all options
+   * ```ts
+   * const result = await client.convert.htmlToMarkdown({
+   *   html: htmlContent,
+   *   baseUrl: "https://shop.example.com",
+   *   options: {
+   *     gfmTables: true,
+   *     removeNoise: true,
+   *     excludeSelectors: [".advertisement", "#sidebar"],
+   *     absoluteUrls: true,
+   *     maxLength: 100000,
+   *     includeImages: true,
+   *     includeLinks: true
+   *   }
+   * });
+   *
+   * // Check metadata
+   * console.log(`Words: ${result.data.metadata.wordCount}`);
+   * console.log(`Links: ${result.data.metadata.linkCount}`);
+   * console.log(`Images: ${result.data.metadata.imageCount}`);
+   * console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
+   *
+   * // Check for warnings
+   * if (result.data.warnings?.length) {
+   *   console.warn("Conversion warnings:", result.data.warnings);
+   * }
+   * ```
+   *
+   * @example Converting scraped HTML
+   * ```ts
+   * // First scrape a page
+   * const scrapeJob = await client.scrape.create({
+   *   url: "https://example.com/article"
+   * });
+   * const scrapeResult = await client.waitForResult(scrapeJob.jobId);
+   *
+   * // Then convert HTML to markdown
+   * const markdown = await client.convert.htmlToMarkdown({
+   *   html: scrapeResult.data.html,
+   *   baseUrl: scrapeResult.data.url,
+   *   options: {
+   *     removeNoise: true,
+   *     onlyMainContent: true
+   *   }
+   * });
+   * ```
+   */
+  async htmlToMarkdown(options) {
+    const body = {
+      html: options.html,
+      baseUrl: options.baseUrl,
+      options: options.options
+    };
+    const result = await post(this.ctx, "/v1/convert", body);
+    return result.data;
+  }
+  /**
+   * Alias for htmlToMarkdown() for convenience
+   *
+   * @example
+   * ```ts
+   * const result = await client.convert.toMarkdown({
+   *   html: "<h1>Hello</h1>"
+   * });
+   * ```
+   */
+  async toMarkdown(options) {
+    return this.htmlToMarkdown(options);
+  }
+  /**
+   * Convert HTML with minimal options (just the HTML content)
+   * Uses all default settings
+   *
+   * @param html - HTML content to convert
+   * @param baseUrl - Optional base URL for resolving relative links
+   * @returns Conversion result
+   *
+   * @example
+   * ```ts
+   * const result = await client.convert.quick(
+   *   "<h1>Title</h1><p>Content</p>",
+   *   "https://example.com"
+   * );
+   * console.log(result.data.markdown);
+   * ```
+   */
+  async quick(html, baseUrl) {
+    return this.htmlToMarkdown({ html, baseUrl });
+  }
+  /**
+   * Convert HTML with noise removal enabled
+   * Removes navigation, footer, ads, scripts, and other clutter
+   *
+   * @param html - HTML content to convert
+   * @param baseUrl - Optional base URL
+   * @returns Conversion result with clean markdown
+   *
+   * @example
+   * ```ts
+   * // Extract just the main content from a messy page
+   * const result = await client.convert.clean(messyHtml, "https://example.com");
+   * console.log(result.data.markdown); // Clean, readable markdown
+   * ```
+   */
+  async clean(html, baseUrl) {
+    return this.htmlToMarkdown({
+      html,
+      baseUrl,
+      options: {
+        removeNoise: true,
+        excludeSelectors: [
+          "nav",
+          "footer",
+          "aside",
+          ".advertisement",
+          ".ad",
+          ".sidebar",
+          "#comments"
+        ]
+      }
+    });
+  }
+};
+
 // src/resources/crawl.ts
 var CRAWL_TEMPLATES = {
   ecommerce: {
@@ -1724,6 +1963,236 @@ var DorkResource = class {
   }
 };
 
+// src/resources/events.ts
+var EventsResource = class {
+  constructor(ctx) {
+    this.ctx = ctx;
+  }
+  /**
+   * Subscribe to all events for authenticated client
+   *
+   * Opens an SSE connection to receive real-time events for all jobs.
+   * Requires an SSE token obtained via client.auth.generateSSEToken().
+   *
+   * Event Types:
+   * - Job lifecycle: job.created, job.queued, job.started, job.progress,
+   *   job.completed, job.failed, job.cancelled
+   * - Job details: job.log, job.metric, job.alert, job.checkpoint
+   * - Deals: deal.found, deal.validated
+   * - System: ping, connection.open, connection.close, error
+   *
+   * Features:
+   * - Automatic reconnection on disconnect
+   * - Event replay via Last-Event-ID
+   * - Keepalive pings every 15 seconds
+   * - Max connection time: 1 hour
+   *
+   * @param token - SSE authentication token from client.auth.generateSSEToken()
+   * @param options - Subscription options (callbacks, reconnection settings)
+   *
+   * @example
+   * ```ts
+   * // Generate token
+   * const { token } = await client.auth.generateSSEToken();
+   *
+   * // Subscribe with event handlers
+   * const eventSource = client.events.subscribe(token, {
+   *   onEvent: (event) => {
+   *     // Handle all events
+   *     console.log('Event:', event.type);
+   *     const data = JSON.parse(event.data);
+   *
+   *     if (data.jobId) {
+   *       console.log(`Job ${data.jobId}:`, data);
+   *     }
+   *   },
+   *   onError: (error) => {
+   *     console.error('SSE error:', error);
+   *   },
+   *   onOpen: () => {
+   *     console.log('SSE connection opened');
+   *   }
+   * });
+   *
+   * // Listen for specific event types
+   * eventSource.addEventListener('job.completed', (event) => {
+   *   const data = JSON.parse(event.data);
+   *   console.log('Job completed:', data);
+   * });
+   *
+   * // Clean up
+   * eventSource.close();
+   * ```
+   */
+  subscribe(token, options) {
+    if (typeof EventSource === "undefined") {
+      throw new Error(
+        "EventSource is not available. SSE subscriptions only work in browsers. For Node.js, use polling via client.status.get() instead."
+      );
+    }
+    const url = new URL("/v1/events", this.ctx.baseUrl);
+    url.searchParams.set("token", token);
+    const eventSource = new EventSource(url.toString());
+    if (options?.onEvent) {
+      eventSource.onmessage = options.onEvent;
+    }
+    if (options?.onError) {
+      eventSource.onerror = (event) => {
+        options.onError(
+          new Error("SSE connection error. Will auto-reconnect if enabled.")
+        );
+      };
+    }
+    if (options?.onOpen) {
+      eventSource.onopen = options.onOpen;
+    }
+    return eventSource;
+  }
+  /**
+   * Subscribe to events for a specific job
+   *
+   * Opens an SSE connection filtered to a single job.
+   * More efficient than global subscription when tracking one job.
+   *
+   * @param jobId - Job ID to subscribe to
+   * @param token - SSE authentication token
+   * @param options - Subscription options
+   *
+   * @example
+   * ```ts
+   * // Start a scrape job
+   * const job = await client.scrape.create({ url: "https://example.com" });
+   *
+   * // Generate SSE token for this job
+   * const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
+   *
+   * // Subscribe to job events
+   * const eventSource = client.events.subscribeToJob(job.jobId, token, {
+   *   onEvent: (event) => {
+   *     const data = JSON.parse(event.data);
+   *     console.log(`[${event.type}]`, data);
+   *   }
+   * });
+   *
+   * // Listen for completion
+   * eventSource.addEventListener('job.completed', (event) => {
+   *   const data = JSON.parse(event.data);
+   *   console.log('Scrape completed!', data.summary);
+   *   eventSource.close();
+   * });
+   *
+   * // Listen for progress
+   * eventSource.addEventListener('job.progress', (event) => {
+   *   const data = JSON.parse(event.data);
+   *   console.log(`Progress: ${data.progress}%`);
+   * });
+   *
+   * // Listen for errors
+   * eventSource.addEventListener('job.failed', (event) => {
+   *   const data = JSON.parse(event.data);
+   *   console.error('Job failed:', data.error);
+   *   eventSource.close();
+   * });
+   * ```
+   */
+  subscribeToJob(jobId, token, options) {
+    if (typeof EventSource === "undefined") {
+      throw new Error(
+        "EventSource is not available. SSE subscriptions only work in browsers. For Node.js, use polling via client.status.get() instead."
+      );
+    }
+    const url = new URL(`/v1/events/${jobId}`, this.ctx.baseUrl);
+    url.searchParams.set("token", token);
+    const eventSource = new EventSource(url.toString());
+    if (options?.onEvent) {
+      eventSource.onmessage = options.onEvent;
+    }
+    if (options?.onError) {
+      eventSource.onerror = (event) => {
+        options.onError(
+          new Error("SSE connection error. Will auto-reconnect if enabled.")
+        );
+      };
+    }
+    if (options?.onOpen) {
+      eventSource.onopen = options.onOpen;
+    }
+    return eventSource;
+  }
+  /**
+   * Helper: Wait for job completion via SSE
+   *
+   * Convenience method that subscribes to a job and resolves when complete.
+   * Automatically handles token generation and cleanup.
+   *
+   * @param jobId - Job ID to wait for
+   * @param onProgress - Optional progress callback
+   *
+   * @example
+   * ```ts
+   * const job = await client.scrape.create({ url: "https://example.com" });
+   *
+   * // Wait for completion with progress updates
+   * const result = await client.events.waitForCompletion(job.jobId, (progress) => {
+   *   console.log(`Progress: ${progress}%`);
+   * });
+   *
+   * console.log('Job completed:', result);
+   * ```
+   */
+  async waitForCompletion(jobId, onProgress) {
+    if (typeof EventSource === "undefined") {
+      throw new Error(
+        "waitForCompletion() only works in browsers. For Node.js, use client.waitForResult() instead."
+      );
+    }
+    return new Promise(async (resolve, reject) => {
+      const tokenResponse = await fetch(
+        `${this.ctx.baseUrl}/v1/auth/sse-token`,
+        {
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${this.ctx.apiKey}`,
+            "Content-Type": "application/json"
+          },
+          body: JSON.stringify({ jobId })
+        }
+      );
+      if (!tokenResponse.ok) {
+        reject(new Error("Failed to generate SSE token"));
+        return;
+      }
+      const { token } = await tokenResponse.json();
+      const eventSource = this.subscribeToJob(jobId, token, {
+        onError: (error) => {
+          eventSource.close();
+          reject(error);
+        }
+      });
+      eventSource.addEventListener("job.progress", (event) => {
+        const data = JSON.parse(event.data);
+        if (onProgress) {
+          onProgress(data.progress);
+        }
+      });
+      eventSource.addEventListener("job.completed", (event) => {
+        const data = JSON.parse(event.data);
+        eventSource.close();
+        resolve(data);
+      });
+      eventSource.addEventListener("job.failed", (event) => {
+        const data = JSON.parse(event.data);
+        eventSource.close();
+        reject(new Error(data.error?.message || "Job failed"));
+      });
+      eventSource.addEventListener("job.cancelled", (event) => {
+        eventSource.close();
+        reject(new Error("Job was cancelled"));
+      });
+    });
+  }
+};
+
 // src/resources/extract.ts
 var ExtractResource = class {
   constructor(ctx) {
@@ -2164,6 +2633,7 @@ var ScrapeResource = class {
   async create(options) {
     const body = {
       url: options.url,
+      noStore: options.noStore,
       detectSignals: options.detectSignals ?? true,
       extractWithAI: options.extractWithAI,
       extractDeal: options.extractDeal,
@@ -2176,7 +2646,10 @@ var ScrapeResource = class {
       excludeSelectors: options.excludeSelectors,
       onlyMainContent: options.onlyMainContent,
       headers: options.headers,
-      timeout: options.timeout
+      timeout: options.timeout,
+      outputMarkdown: options.outputMarkdown,
+      markdownBaseUrl: options.markdownBaseUrl,
+      actions: options.actions
     };
     const result = await post(this.ctx, "/v1/scrape", body);
     return result.data;
@@ -2252,7 +2725,8 @@ var ScrapeResource = class {
    *     { url: "https://shop1.com/product1" },
    *     { url: "https://shop2.com/deal", extractDeal: true }
    *   ],
-   *   defaults: { detectSignals: true }
+   *   defaults: { detectSignals: true },
+   *   ignoreInvalidURLs: true
    * });
    * console.log(batch.batchId, batch.results);
    * ```
@@ -2263,7 +2737,8 @@ var ScrapeResource = class {
       defaults: options.defaults,
       webhookUrl: options.webhookUrl,
       priority: options.priority,
-      delayMs: options.delay
+      delayMs: options.delayMs,
+      ignoreInvalidURLs: options.ignoreInvalidURLs
     };
     const result = await post(
       this.ctx,
@@ -2313,6 +2788,57 @@ var ScrapeResource = class {
   }
 };
 
+// src/resources/screenshots.ts
+var ScreenshotsResource = class {
+  constructor(ctx) {
+    this.ctx = ctx;
+  }
+  /**
+   * Refresh a signed URL before expiration
+   *
+   * @example
+   * ```ts
+   * const refreshed = await client.screenshots.refresh({
+   *   path: "job_abc123/1234567890_nanoid_example.png",
+   *   ttl: 604800  // 7 days
+   * });
+   * console.log(refreshed.url);           // New signed URL
+   * console.log(refreshed.expiresAt);     // "2026-01-25T12:00:00Z"
+   * console.log(refreshed.tierLimits);    // { min: 3600, max: 604800, default: 604800 }
+   * ```
+   */
+  async refresh(options) {
+    const result = await post(
+      this.ctx,
+      "/v1/screenshots/refresh",
+      {
+        path: options.path,
+        ttl: options.ttl,
+        bucket: options.bucket
+      }
+    );
+    return result.data;
+  }
+  /**
+   * Get TTL limits for the current tier
+   *
+   * @example
+   * ```ts
+   * const limits = await client.screenshots.getLimits();
+   * console.log(limits.tier);                    // "pro"
+   * console.log(limits.limits.max);              // 604800 (7 days in seconds)
+   * console.log(limits.formattedLimits.max);     // "7 days"
+   * ```
+   */
+  async getLimits() {
+    const result = await get(
+      this.ctx,
+      "/v1/screenshots/limits"
+    );
+    return result.data;
+  }
+};
+
 // src/resources/search.ts
 var SearchResource = class {
   constructor(ctx) {
@@ -2325,7 +2851,7 @@ var SearchResource = class {
    * ```ts
    * const result = await client.search.create({
    *   query: "laptop deals black friday",
-   *   maxResults: 20,
+   *   limit: 20,
    *   useDealScoring: true
    * });
    * ```
@@ -2333,9 +2859,9 @@ var SearchResource = class {
   async create(options) {
     const body = {
       query: options.query,
-      limit: options.maxResults,
-      scrapeResults: options.autoScrape,
-      maxScrapeResults: options.autoScrapeLimit,
+      limit: options.limit,
+      scrapeResults: options.scrapeResults,
+      maxScrapeResults: options.maxScrapeResults,
       useAiOptimization: options.useAiOptimization,
       aiProvider: options.aiProvider,
       aiModel: options.aiModel,
@@ -2389,7 +2915,7 @@ var SearchResource = class {
    * @example
    * ```ts
    * const result = await client.search.andScrape("promo codes", {
-   *   autoScrapeLimit: 5
+   *   maxScrapeResults: 5
    * });
    * console.log(result.data.scrapedJobIds);
    * ```
@@ -2397,7 +2923,7 @@ var SearchResource = class {
   async andScrape(query, options) {
     return this.create({
       query,
-      autoScrape: true,
+      scrapeResults: true,
       ...options
     });
   }
@@ -2811,6 +3337,20 @@ var DealCrawl = class {
    * ```
    */
   dork;
+  /**
+   * Convert resource - HTML to Markdown conversion
+   *
+   * @example
+   * ```ts
+   * const result = await client.convert.htmlToMarkdown({
+   *   html: "<h1>Title</h1><p>Content</p>",
+   *   baseUrl: "https://example.com",
+   *   options: { removeNoise: true }
+   * });
+   * console.log(result.data.markdown);
+   * ```
+   */
+  convert;
   /**
    * Agent resource - AI-powered autonomous web navigation
    *
@@ -2892,6 +3432,80 @@ var DealCrawl = class {
    * ```
    */
   account;
+  /**
+   * Screenshots resource - Screenshot signed URL management
+   *
+   * @example
+   * ```ts
+   * // Refresh a signed URL before expiration
+   * const refreshed = await client.screenshots.refresh({
+   *   path: "job_abc123/1234567890_nanoid_example.png",
+   *   ttl: 604800  // 7 days
+   * });
+   *
+   * // Get tier-specific TTL limits
+   * const limits = await client.screenshots.getLimits();
+   * console.log(limits.formattedLimits.max);  // "7 days"
+   * ```
+   */
+  screenshots;
+  /**
+   * Auth resource - SSE (Server-Sent Events) authentication
+   *
+   * @example
+   * ```ts
+   * // Generate SSE token for browser EventSource
+   * const { token, expiresAt } = await client.auth.generateSSEToken();
+   *
+   * // Use in browser
+   * const eventSource = new EventSource(`/v1/events?token=${token}`);
+   *
+   * // Generate token for specific job
+   * const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
+   *
+   * // Check connection limits
+   * const limits = await client.auth.getLimits();
+   * console.log(`Available connections: ${limits.sse.available}`);
+   * ```
+   */
+  auth;
+  /**
+   * Events resource - Real-time SSE event streaming (Browser only)
+   *
+   * IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
+   *
+   * @example Browser Usage
+   * ```ts
+   * // 1. Generate SSE token
+   * const { token } = await client.auth.generateSSEToken();
+   *
+   * // 2. Subscribe to all events
+   * const eventSource = client.events.subscribe(token, {
+   *   onEvent: (event) => {
+   *     console.log('Event:', event.type, JSON.parse(event.data));
+   *   }
+   * });
+   *
+   * // 3. Or subscribe to specific job
+   * const jobEvents = client.events.subscribeToJob('job_123', token, {
+   *   onEvent: (event) => {
+   *     const data = JSON.parse(event.data);
+   *     console.log(`Progress: ${data.progress}%`);
+   *   }
+   * });
+   *
+   * // 4. Listen for specific events
+   * eventSource.addEventListener('job.completed', (event) => {
+   *   const data = JSON.parse(event.data);
+   *   console.log('Job completed!', data.summary);
+   *   eventSource.close();
+   * });
+   *
+   * // 5. Clean up
+   * eventSource.close();
+   * ```
+   */
+  events;
   // ============================================
   // CONSTRUCTOR
   // ============================================
@@ -2942,12 +3556,16 @@ var DealCrawl = class {
     this.crawl = new CrawlResource(this.ctx);
     this.extract = new ExtractResource(this.ctx);
     this.dork = new DorkResource(this.ctx);
+    this.convert = new ConvertResource(this.ctx);
     this.agent = new AgentResource(this.ctx);
     this.status = new StatusResource(this.ctx);
     this.data = new DataResource(this.ctx);
     this.webhooks = new WebhooksResource(this.ctx);
     this.keys = new KeysResource(this.ctx);
     this.account = new AccountResource(this.ctx);
+    this.screenshots = new ScreenshotsResource(this.ctx);
+    this.auth = new AuthResource(this.ctx);
+    this.events = new EventsResource(this.ctx);
   }
   // ============================================
   // POLLING METHODS
@@ -3093,6 +3711,8 @@ var DealCrawl = class {
 
 exports.AccountResource = AccountResource;
 exports.AgentResource = AgentResource;
+exports.AuthResource = AuthResource;
+exports.ConvertResource = ConvertResource;
 exports.CrawlResource = CrawlResource;
 exports.DEFAULT_CONFIG = DEFAULT_CONFIG;
 exports.DataResource = DataResource;
@@ -3101,9 +3721,11 @@ exports.DealCrawlError = DealCrawlError;
 exports.DorkResource = DorkResource;
 exports.ERROR_CODES = ERROR_CODES;
 exports.ERROR_MESSAGES = ERROR_MESSAGES;
+exports.EventsResource = EventsResource;
 exports.ExtractResource = ExtractResource;
 exports.KeysResource = KeysResource;
 exports.ScrapeResource = ScrapeResource;
+exports.ScreenshotsResource = ScreenshotsResource;
 exports.SearchResource = SearchResource;
 exports.StatusResource = StatusResource;
 exports.WebhooksResource = WebhooksResource;