clearscrape 1.0.0

package/dist/index.js

@@ -0,0 +1,356 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  ClearScrape: () => ClearScrape,
+  ClearScrapeError: () => ClearScrapeError,
+  InsufficientCreditsError: () => InsufficientCreditsError,
+  RateLimitError: () => RateLimitError,
+  default: () => ClearScrape
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/types.ts
+var ClearScrapeError = class extends Error {
+  constructor(message, statusCode, response) {
+    super(message);
+    this.name = "ClearScrapeError";
+    this.statusCode = statusCode;
+    this.response = response;
+  }
+};
+var InsufficientCreditsError = class extends ClearScrapeError {
+  constructor(message, required) {
+    super(message, 402);
+    this.name = "InsufficientCreditsError";
+    this.required = required;
+  }
+};
+var RateLimitError = class extends ClearScrapeError {
+  constructor(message) {
+    super(message, 429);
+    this.name = "RateLimitError";
+  }
+};
+
+// src/client.ts
+var DEFAULT_BASE_URL = "https://api.clearscrape.io";
+var DEFAULT_TIMEOUT = 6e4;
+var DEFAULT_RETRIES = 3;
+var ClearScrape = class {
+  constructor(config) {
+    if (!config.apiKey) {
+      throw new Error("API key is required");
+    }
+    this.apiKey = config.apiKey;
+    this.baseUrl = config.baseUrl || DEFAULT_BASE_URL;
+    this.timeout = config.timeout || DEFAULT_TIMEOUT;
+    this.retries = config.retries ?? DEFAULT_RETRIES;
+  }
+  /**
+   * Scrape a URL and return the HTML content
+   *
+   * @param options - Scraping options
+   * @returns Promise resolving to scrape response
+   *
+   * @example
+   * ```typescript
+   * // Basic scrape
+   * const result = await client.scrape({ url: 'https://example.com' });
+   *
+   * // With JavaScript rendering
+   * const result = await client.scrape({
+   *   url: 'https://example.com',
+   *   jsRender: true,
+   *   waitFor: '.content'
+   * });
+   *
+   * // With premium proxy and country targeting
+   * const result = await client.scrape({
+   *   url: 'https://example.com',
+   *   premiumProxy: true,
+   *   proxyCountry: 'us'
+   * });
+   * ```
+   */
+  async scrape(options) {
+    const payload = this.buildPayload(options);
+    return this.makeRequest("/api/scrape", payload);
+  }
+  /**
+   * Scrape a URL and return only the HTML content
+   *
+   * @param url - URL to scrape
+   * @param options - Additional scraping options
+   * @returns Promise resolving to HTML string
+   */
+  async getHtml(url, options) {
+    const result = await this.scrape({ url, ...options });
+    return result.data.html;
+  }
+  /**
+   * Scrape a URL and return only the text content
+   *
+   * @param url - URL to scrape
+   * @param options - Additional scraping options
+   * @returns Promise resolving to text string
+   */
+  async getText(url, options) {
+    const result = await this.scrape({ url, ...options });
+    return result.data.text || "";
+  }
+  /**
+   * Take a screenshot of a URL
+   *
+   * @param url - URL to screenshot
+   * @param options - Additional options
+   * @returns Promise resolving to base64 encoded screenshot
+   *
+   * @example
+   * ```typescript
+   * const screenshot = await client.screenshot('https://example.com');
+   * // Save to file
+   * fs.writeFileSync('screenshot.png', Buffer.from(screenshot, 'base64'));
+   * ```
+   */
+  async screenshot(url, options) {
+    const result = await this.scrape({
+      url,
+      ...options,
+      jsRender: true,
+      screenshot: true
+    });
+    if (!result.data.screenshot) {
+      throw new ClearScrapeError("Screenshot not returned", 500);
+    }
+    const base64 = result.data.screenshot.replace(/^data:image\/\w+;base64,/, "");
+    return base64;
+  }
+  /**
+   * Scrape using a domain-specific extractor (Amazon, Walmart, Google, etc.)
+   *
+   * @param url - URL to scrape
+   * @param domain - Domain extractor to use
+   * @returns Promise resolving to extracted data
+   *
+   * @example
+   * ```typescript
+   * // Extract Amazon product data
+   * const product = await client.extract(
+   *   'https://www.amazon.com/dp/B09V3KXJPB',
+   *   'amazon'
+   * );
+   * console.log(product.title, product.price);
+   *
+   * // Extract Google SERP data
+   * const serp = await client.extract(
+   *   'https://www.google.com/search?q=best+laptops',
+   *   'google'
+   * );
+   * console.log(serp.organicResults);
+   * ```
+   */
+  async extract(url, domain) {
+    const result = await this.scrape({ url, domain });
+    if (!result.data.extracted) {
+      throw new ClearScrapeError("No extracted data returned", 500);
+    }
+    return result.data.extracted;
+  }
+  /**
+   * Get proxy configuration for the residential proxy service
+   *
+   * @param options - Proxy options
+   * @returns Proxy configuration object
+   *
+   * @example
+   * ```typescript
+   * // Basic proxy config
+   * const proxy = client.getProxyConfig();
+   * // { host: 'proxy.clearscrape.io', port: 8000, username: '...', password: '...' }
+   *
+   * // With country targeting
+   * const proxy = client.getProxyConfig({ country: 'us' });
+   *
+   * // With session sticky IP
+   * const proxy = client.getProxyConfig({ session: 'my-session-123' });
+   * ```
+   */
+  getProxyConfig(options) {
+    let username = this.apiKey;
+    if (options?.country) {
+      username += `-country-${options.country}`;
+    }
+    if (options?.session) {
+      username += `-session-${options.session}`;
+    }
+    return {
+      host: "proxy.clearscrape.io",
+      port: 8e3,
+      username,
+      password: this.apiKey
+    };
+  }
+  /**
+   * Get proxy URL string for use with HTTP clients
+   *
+   * @param options - Proxy options
+   * @returns Proxy URL string
+   *
+   * @example
+   * ```typescript
+   * const proxyUrl = client.getProxyUrl({ country: 'us' });
+   * // 'http://apikey-country-us:apikey@proxy.clearscrape.io:8000'
+   *
+   * // Use with axios
+   * const HttpsProxyAgent = require('https-proxy-agent');
+   * const agent = new HttpsProxyAgent(client.getProxyUrl());
+   * axios.get(url, { httpsAgent: agent });
+   * ```
+   */
+  getProxyUrl(options) {
+    const config = this.getProxyConfig(options);
+    return `http://${config.username}:${config.password}@${config.host}:${config.port}`;
+  }
+  /**
+   * Get WebSocket URL for Scraping Browser (Playwright/Puppeteer)
+   *
+   * @param options - Browser options
+   * @returns WebSocket URL string
+   *
+   * @example
+   * ```typescript
+   * // Use with Playwright
+   * const { chromium } = require('playwright');
+   * const browser = await chromium.connectOverCDP(client.getBrowserWsUrl());
+   *
+   * // Use with Puppeteer
+   * const puppeteer = require('puppeteer-core');
+   * const browser = await puppeteer.connect({
+   *   browserWSEndpoint: client.getBrowserWsUrl()
+   * });
+   *
+   * // With country targeting
+   * const wsUrl = client.getBrowserWsUrl({ proxyCountry: 'gb' });
+   * ```
+   */
+  getBrowserWsUrl(options) {
+    let url = `wss://browser.clearscrape.io?apiKey=${this.apiKey}`;
+    if (options?.proxyCountry) {
+      url += `&proxy_country=${options.proxyCountry}`;
+    }
+    return url;
+  }
+  /**
+   * Build the API request payload
+   */
+  buildPayload(options) {
+    const payload = {
+      url: options.url
+    };
+    if (options.method) payload.method = options.method;
+    if (options.jsRender) payload.js_render = options.jsRender;
+    if (options.premiumProxy) payload.premium_proxy = options.premiumProxy;
+    if (options.antibot) payload.antibot = options.antibot;
+    if (options.proxyCountry) payload.proxy_country = options.proxyCountry;
+    if (options.waitFor) payload.wait_for = options.waitFor;
+    if (options.wait) payload.wait = options.wait;
+    if (options.autoScroll) payload.auto_scroll = options.autoScroll;
+    if (options.screenshot) payload.screenshot = options.screenshot;
+    if (options.screenshotSelector) payload.screenshot_selector = options.screenshotSelector;
+    if (options.headers) payload.headers = options.headers;
+    if (options.body) payload.body = options.body;
+    if (options.domain) payload.domain = options.domain;
+    return payload;
+  }
+  /**
+   * Make an API request with retries
+   */
+  async makeRequest(endpoint, payload, attempt = 1) {
+    const url = `${this.baseUrl}${endpoint}`;
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const response = await fetch(url, {
+        method: "POST",
+        headers: {
+          "X-API-Key": this.apiKey,
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify(payload),
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      const data = await response.json();
+      if (!response.ok) {
+        return this.handleError(response.status, data, payload, attempt);
+      }
+      return data;
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === "AbortError") {
+        throw new ClearScrapeError("Request timeout", 408);
+      }
+      if (attempt < this.retries) {
+        const delay = Math.pow(2, attempt) * 1e3;
+        await this.sleep(delay);
+        return this.makeRequest(endpoint, payload, attempt + 1);
+      }
+      throw new ClearScrapeError(
+        error instanceof Error ? error.message : "Unknown error",
+        500
+      );
+    }
+  }
+  /**
+   * Handle API errors
+   */
+  async handleError(statusCode, response, payload, attempt) {
+    if (statusCode >= 400 && statusCode < 500 && statusCode !== 429) {
+      if (statusCode === 402 && response.required) {
+        throw new InsufficientCreditsError(response.message, response.required);
+      }
+      throw new ClearScrapeError(response.message || response.error, statusCode, response);
+    }
+    if (attempt < this.retries) {
+      const delay = statusCode === 429 ? 5e3 : Math.pow(2, attempt) * 1e3;
+      await this.sleep(delay);
+      return this.makeRequest("/api/scrape", payload, attempt + 1);
+    }
+    if (statusCode === 429) {
+      throw new RateLimitError(response.message || "Rate limit exceeded");
+    }
+    throw new ClearScrapeError(response.message || response.error, statusCode, response);
+  }
+  /**
+   * Sleep for a specified duration
+   */
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ClearScrape,
+  ClearScrapeError,
+  InsufficientCreditsError,
+  RateLimitError
+});

package/dist/index.mjs

@@ -0,0 +1,326 @@
+// src/types.ts
+var ClearScrapeError = class extends Error {
+  constructor(message, statusCode, response) {
+    super(message);
+    this.name = "ClearScrapeError";
+    this.statusCode = statusCode;
+    this.response = response;
+  }
+};
+var InsufficientCreditsError = class extends ClearScrapeError {
+  constructor(message, required) {
+    super(message, 402);
+    this.name = "InsufficientCreditsError";
+    this.required = required;
+  }
+};
+var RateLimitError = class extends ClearScrapeError {
+  constructor(message) {
+    super(message, 429);
+    this.name = "RateLimitError";
+  }
+};
+
+// src/client.ts
+var DEFAULT_BASE_URL = "https://api.clearscrape.io";
+var DEFAULT_TIMEOUT = 6e4;
+var DEFAULT_RETRIES = 3;
+var ClearScrape = class {
+  constructor(config) {
+    if (!config.apiKey) {
+      throw new Error("API key is required");
+    }
+    this.apiKey = config.apiKey;
+    this.baseUrl = config.baseUrl || DEFAULT_BASE_URL;
+    this.timeout = config.timeout || DEFAULT_TIMEOUT;
+    this.retries = config.retries ?? DEFAULT_RETRIES;
+  }
+  /**
+   * Scrape a URL and return the HTML content
+   *
+   * @param options - Scraping options
+   * @returns Promise resolving to scrape response
+   *
+   * @example
+   * ```typescript
+   * // Basic scrape
+   * const result = await client.scrape({ url: 'https://example.com' });
+   *
+   * // With JavaScript rendering
+   * const result = await client.scrape({
+   *   url: 'https://example.com',
+   *   jsRender: true,
+   *   waitFor: '.content'
+   * });
+   *
+   * // With premium proxy and country targeting
+   * const result = await client.scrape({
+   *   url: 'https://example.com',
+   *   premiumProxy: true,
+   *   proxyCountry: 'us'
+   * });
+   * ```
+   */
+  async scrape(options) {
+    const payload = this.buildPayload(options);
+    return this.makeRequest("/api/scrape", payload);
+  }
+  /**
+   * Scrape a URL and return only the HTML content
+   *
+   * @param url - URL to scrape
+   * @param options - Additional scraping options
+   * @returns Promise resolving to HTML string
+   */
+  async getHtml(url, options) {
+    const result = await this.scrape({ url, ...options });
+    return result.data.html;
+  }
+  /**
+   * Scrape a URL and return only the text content
+   *
+   * @param url - URL to scrape
+   * @param options - Additional scraping options
+   * @returns Promise resolving to text string
+   */
+  async getText(url, options) {
+    const result = await this.scrape({ url, ...options });
+    return result.data.text || "";
+  }
+  /**
+   * Take a screenshot of a URL
+   *
+   * @param url - URL to screenshot
+   * @param options - Additional options
+   * @returns Promise resolving to base64 encoded screenshot
+   *
+   * @example
+   * ```typescript
+   * const screenshot = await client.screenshot('https://example.com');
+   * // Save to file
+   * fs.writeFileSync('screenshot.png', Buffer.from(screenshot, 'base64'));
+   * ```
+   */
+  async screenshot(url, options) {
+    const result = await this.scrape({
+      url,
+      ...options,
+      jsRender: true,
+      screenshot: true
+    });
+    if (!result.data.screenshot) {
+      throw new ClearScrapeError("Screenshot not returned", 500);
+    }
+    const base64 = result.data.screenshot.replace(/^data:image\/\w+;base64,/, "");
+    return base64;
+  }
+  /**
+   * Scrape using a domain-specific extractor (Amazon, Walmart, Google, etc.)
+   *
+   * @param url - URL to scrape
+   * @param domain - Domain extractor to use
+   * @returns Promise resolving to extracted data
+   *
+   * @example
+   * ```typescript
+   * // Extract Amazon product data
+   * const product = await client.extract(
+   *   'https://www.amazon.com/dp/B09V3KXJPB',
+   *   'amazon'
+   * );
+   * console.log(product.title, product.price);
+   *
+   * // Extract Google SERP data
+   * const serp = await client.extract(
+   *   'https://www.google.com/search?q=best+laptops',
+   *   'google'
+   * );
+   * console.log(serp.organicResults);
+   * ```
+   */
+  async extract(url, domain) {
+    const result = await this.scrape({ url, domain });
+    if (!result.data.extracted) {
+      throw new ClearScrapeError("No extracted data returned", 500);
+    }
+    return result.data.extracted;
+  }
+  /**
+   * Get proxy configuration for the residential proxy service
+   *
+   * @param options - Proxy options
+   * @returns Proxy configuration object
+   *
+   * @example
+   * ```typescript
+   * // Basic proxy config
+   * const proxy = client.getProxyConfig();
+   * // { host: 'proxy.clearscrape.io', port: 8000, username: '...', password: '...' }
+   *
+   * // With country targeting
+   * const proxy = client.getProxyConfig({ country: 'us' });
+   *
+   * // With session sticky IP
+   * const proxy = client.getProxyConfig({ session: 'my-session-123' });
+   * ```
+   */
+  getProxyConfig(options) {
+    let username = this.apiKey;
+    if (options?.country) {
+      username += `-country-${options.country}`;
+    }
+    if (options?.session) {
+      username += `-session-${options.session}`;
+    }
+    return {
+      host: "proxy.clearscrape.io",
+      port: 8e3,
+      username,
+      password: this.apiKey
+    };
+  }
+  /**
+   * Get proxy URL string for use with HTTP clients
+   *
+   * @param options - Proxy options
+   * @returns Proxy URL string
+   *
+   * @example
+   * ```typescript
+   * const proxyUrl = client.getProxyUrl({ country: 'us' });
+   * // 'http://apikey-country-us:apikey@proxy.clearscrape.io:8000'
+   *
+   * // Use with axios
+   * const HttpsProxyAgent = require('https-proxy-agent');
+   * const agent = new HttpsProxyAgent(client.getProxyUrl());
+   * axios.get(url, { httpsAgent: agent });
+   * ```
+   */
+  getProxyUrl(options) {
+    const config = this.getProxyConfig(options);
+    return `http://${config.username}:${config.password}@${config.host}:${config.port}`;
+  }
+  /**
+   * Get WebSocket URL for Scraping Browser (Playwright/Puppeteer)
+   *
+   * @param options - Browser options
+   * @returns WebSocket URL string
+   *
+   * @example
+   * ```typescript
+   * // Use with Playwright
+   * const { chromium } = require('playwright');
+   * const browser = await chromium.connectOverCDP(client.getBrowserWsUrl());
+   *
+   * // Use with Puppeteer
+   * const puppeteer = require('puppeteer-core');
+   * const browser = await puppeteer.connect({
+   *   browserWSEndpoint: client.getBrowserWsUrl()
+   * });
+   *
+   * // With country targeting
+   * const wsUrl = client.getBrowserWsUrl({ proxyCountry: 'gb' });
+   * ```
+   */
+  getBrowserWsUrl(options) {
+    let url = `wss://browser.clearscrape.io?apiKey=${this.apiKey}`;
+    if (options?.proxyCountry) {
+      url += `&proxy_country=${options.proxyCountry}`;
+    }
+    return url;
+  }
+  /**
+   * Build the API request payload
+   */
+  buildPayload(options) {
+    const payload = {
+      url: options.url
+    };
+    if (options.method) payload.method = options.method;
+    if (options.jsRender) payload.js_render = options.jsRender;
+    if (options.premiumProxy) payload.premium_proxy = options.premiumProxy;
+    if (options.antibot) payload.antibot = options.antibot;
+    if (options.proxyCountry) payload.proxy_country = options.proxyCountry;
+    if (options.waitFor) payload.wait_for = options.waitFor;
+    if (options.wait) payload.wait = options.wait;
+    if (options.autoScroll) payload.auto_scroll = options.autoScroll;
+    if (options.screenshot) payload.screenshot = options.screenshot;
+    if (options.screenshotSelector) payload.screenshot_selector = options.screenshotSelector;
+    if (options.headers) payload.headers = options.headers;
+    if (options.body) payload.body = options.body;
+    if (options.domain) payload.domain = options.domain;
+    return payload;
+  }
+  /**
+   * Make an API request with retries
+   */
+  async makeRequest(endpoint, payload, attempt = 1) {
+    const url = `${this.baseUrl}${endpoint}`;
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const response = await fetch(url, {
+        method: "POST",
+        headers: {
+          "X-API-Key": this.apiKey,
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify(payload),
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      const data = await response.json();
+      if (!response.ok) {
+        return this.handleError(response.status, data, payload, attempt);
+      }
+      return data;
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === "AbortError") {
+        throw new ClearScrapeError("Request timeout", 408);
+      }
+      if (attempt < this.retries) {
+        const delay = Math.pow(2, attempt) * 1e3;
+        await this.sleep(delay);
+        return this.makeRequest(endpoint, payload, attempt + 1);
+      }
+      throw new ClearScrapeError(
+        error instanceof Error ? error.message : "Unknown error",
+        500
+      );
+    }
+  }
+  /**
+   * Handle API errors
+   */
+  async handleError(statusCode, response, payload, attempt) {
+    if (statusCode >= 400 && statusCode < 500 && statusCode !== 429) {
+      if (statusCode === 402 && response.required) {
+        throw new InsufficientCreditsError(response.message, response.required);
+      }
+      throw new ClearScrapeError(response.message || response.error, statusCode, response);
+    }
+    if (attempt < this.retries) {
+      const delay = statusCode === 429 ? 5e3 : Math.pow(2, attempt) * 1e3;
+      await this.sleep(delay);
+      return this.makeRequest("/api/scrape", payload, attempt + 1);
+    }
+    if (statusCode === 429) {
+      throw new RateLimitError(response.message || "Rate limit exceeded");
+    }
+    throw new ClearScrapeError(response.message || response.error, statusCode, response);
+  }
+  /**
+   * Sleep for a specified duration
+   */
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+};
+export {
+  ClearScrape,
+  ClearScrapeError,
+  InsufficientCreditsError,
+  RateLimitError,
+  ClearScrape as default
+};