@crawlgate/sdk 1.0.0

package/dist/index.js

@@ -0,0 +1,1255 @@
+// src/utils/httpClient.ts
+import axios from "axios";
+
+// src/errors.ts
+var CrawlGateError = class _CrawlGateError extends Error {
+  /**
+   * HTTP status code (if applicable)
+   */
+  statusCode;
+  /**
+   * Error code for programmatic handling
+   */
+  code;
+  /**
+   * Additional error details
+   */
+  details;
+  constructor(message, statusCode, code, details) {
+    super(message);
+    this.name = "CrawlGateError";
+    this.statusCode = statusCode;
+    this.code = code;
+    this.details = details;
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, _CrawlGateError);
+    }
+  }
+};
+var AuthenticationError = class extends CrawlGateError {
+  constructor(message = "Invalid API Key") {
+    super(message, 401, "AUTHENTICATION_ERROR");
+    this.name = "AuthenticationError";
+  }
+};
+var ValidationError = class extends CrawlGateError {
+  constructor(message, details) {
+    super(message, 400, "VALIDATION_ERROR", details);
+    this.name = "ValidationError";
+  }
+};
+var JobTimeoutError = class extends CrawlGateError {
+  /**
+   * Job ID that timed out
+   */
+  jobId;
+  /**
+   * Timeout duration in seconds
+   */
+  timeoutSeconds;
+  constructor(jobId, timeoutSeconds) {
+    super(
+      `Crawl job ${jobId} did not complete within ${timeoutSeconds} seconds`,
+      void 0,
+      "JOB_TIMEOUT"
+    );
+    this.name = "JobTimeoutError";
+    this.jobId = jobId;
+    this.timeoutSeconds = timeoutSeconds;
+  }
+};
+var ServiceUnavailableError = class extends CrawlGateError {
+  constructor(message = "Service temporarily unavailable") {
+    super(message, 503, "SERVICE_UNAVAILABLE");
+    this.name = "ServiceUnavailableError";
+  }
+};
+var RateLimitError = class extends CrawlGateError {
+  /**
+   * Time to wait before retrying (in seconds)
+   */
+  retryAfter;
+  constructor(message = "Rate limit exceeded", retryAfter) {
+    super(message, 429, "RATE_LIMIT_EXCEEDED");
+    this.name = "RateLimitError";
+    this.retryAfter = retryAfter;
+  }
+};
+var ExtractionError = class extends CrawlGateError {
+  /**
+   * Provider that failed
+   */
+  provider;
+  constructor(message, provider) {
+    super(message, void 0, "EXTRACTION_ERROR");
+    this.name = "ExtractionError";
+    this.provider = provider;
+  }
+};
+function parseApiError(status, data) {
+  const message = data.error || data.message || "Unknown error";
+  switch (status) {
+    case 400:
+      throw new ValidationError(message, data.details);
+    case 401:
+      throw new AuthenticationError(message);
+    case 429:
+      throw new RateLimitError(message);
+    case 502:
+    case 503:
+      throw new ServiceUnavailableError(message);
+    default:
+      throw new CrawlGateError(message, status, void 0, data.details);
+  }
+}
+
+// src/utils/httpClient.ts
+var HttpClient = class {
+  instance;
+  apiKey;
+  apiUrl;
+  maxRetries;
+  backoffFactor;
+  constructor(options) {
+    this.apiKey = options.apiKey;
+    this.apiUrl = options.apiUrl.replace(/\/$/, "");
+    this.maxRetries = options.maxRetries ?? 3;
+    this.backoffFactor = options.backoffFactor ?? 0.5;
+    this.instance = axios.create({
+      baseURL: this.apiUrl,
+      timeout: options.timeoutMs ?? 9e4,
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": this.apiKey
+      }
+    });
+  }
+  /**
+   * Get the configured API URL
+   */
+  getApiUrl() {
+    return this.apiUrl;
+  }
+  /**
+   * Get the configured API key
+   */
+  getApiKey() {
+    return this.apiKey;
+  }
+  /**
+   * Sleep for specified seconds
+   */
+  sleep(seconds) {
+    return new Promise((resolve) => setTimeout(resolve, seconds * 1e3));
+  }
+  /**
+   * Check if error is retryable
+   */
+  isRetryableError(status) {
+    return status === 502 || status === 503 || status === 429;
+  }
+  /**
+   * Make HTTP request with retry logic
+   */
+  async request(config) {
+    let lastError;
+    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+      try {
+        if (config.method && ["post", "put", "patch"].includes(config.method.toLowerCase())) {
+          const data = config.data ?? {};
+          config.data = { ...data, origin: "crawlgate-sdk" };
+          if (typeof data.timeout === "number") {
+            config.timeout = data.timeout + 5e3;
+          }
+        }
+        const response = await this.instance.request(config);
+        if (this.isRetryableError(response.status) && attempt < this.maxRetries - 1) {
+          await this.sleep(this.backoffFactor * Math.pow(2, attempt));
+          continue;
+        }
+        return response;
+      } catch (err) {
+        const axiosError = err;
+        lastError = err instanceof Error ? err : new Error(String(axiosError?.message));
+        const status = axiosError?.response?.status;
+        if (this.isRetryableError(status) && attempt < this.maxRetries - 1) {
+          await this.sleep(this.backoffFactor * Math.pow(2, attempt));
+          continue;
+        }
+        if (axiosError?.response) {
+          parseApiError(
+            axiosError.response.status,
+            axiosError.response.data
+          );
+        }
+        throw lastError;
+      }
+    }
+    throw lastError ?? new CrawlGateError("Unexpected HTTP client error");
+  }
+  /**
+   * Make POST request
+   */
+  async post(endpoint, body, headers) {
+    return this.request({
+      method: "post",
+      url: endpoint,
+      data: body,
+      headers
+    });
+  }
+  /**
+   * Make GET request
+   */
+  async get(endpoint, headers) {
+    return this.request({
+      method: "get",
+      url: endpoint,
+      headers
+    });
+  }
+  /**
+   * Make DELETE request
+   */
+  async delete(endpoint, headers) {
+    return this.request({
+      method: "delete",
+      url: endpoint,
+      headers
+    });
+  }
+};
+
+// src/methods/scrape.ts
+import { zodToJsonSchema } from "zod-to-json-schema";
+function convertSchema(schema) {
+  if (schema && typeof schema === "object" && "_def" in schema) {
+    return zodToJsonSchema(schema);
+  }
+  return schema;
+}
+function buildScrapeBody(url, options) {
+  const body = { url };
+  if (options?.engine) {
+    body.engine = options.engine;
+  }
+  if (options?.formats) {
+    body.formats = options.formats;
+  }
+  if (options?.onlyMainContent !== void 0) {
+    body.onlyMainContent = options.onlyMainContent;
+  }
+  if (options?.excludeTags) {
+    body.excludeTags = options.excludeTags;
+  }
+  if (options?.waitFor !== void 0) {
+    body.waitFor = options.waitFor;
+  }
+  if (options?.timeout !== void 0) {
+    body.timeout = options.timeout;
+  }
+  if (options?.proxy) {
+    body.proxy = options.proxy;
+  }
+  if (options?.projectId) {
+    body.project_id = options.projectId;
+  }
+  if (options?.extract) {
+    body.extract = {
+      schema: convertSchema(options.extract.schema),
+      systemPrompt: options.extract.systemPrompt,
+      provider: options.extract.provider,
+      enableFallback: options.extract.enableFallback
+    };
+    Object.keys(body.extract).forEach((key) => {
+      if (body.extract[key] === void 0) {
+        delete body.extract[key];
+      }
+    });
+  }
+  return body;
+}
+async function scrape(http, url, options) {
+  const body = buildScrapeBody(url, options);
+  const response = await http.post("/v1/scrape", body);
+  if (!response.data.success) {
+    throw new CrawlGateError(
+      response.data.error || "Scrape failed",
+      void 0,
+      "SCRAPE_ERROR"
+    );
+  }
+  if (!response.data.data) {
+    throw new CrawlGateError("No data returned from scrape", void 0, "NO_DATA");
+  }
+  const document = {
+    ...response.data.data
+  };
+  return document;
+}
+
+// src/methods/crawl.ts
+function buildCrawlBody(url, options) {
+  const body = { url };
+  if (options?.engine) {
+    body.engine = options.engine;
+  }
+  if (options?.limit !== void 0) {
+    body.limit = options.limit;
+  }
+  if (options?.formats) {
+    body.formats = options.formats;
+  }
+  if (options?.onlyMainContent !== void 0) {
+    body.onlyMainContent = options.onlyMainContent;
+  }
+  if (options?.excludeTags) {
+    body.excludeTags = options.excludeTags;
+  }
+  if (options?.proxy) {
+    body.proxy = options.proxy;
+  }
+  if (options?.projectId) {
+    body.project_id = options.projectId;
+  }
+  return body;
+}
+async function startCrawl(http, url, options) {
+  const body = buildCrawlBody(url, options);
+  const response = await http.post("/v1/crawl", body);
+  if (!response.data.success && !response.data.id) {
+    throw new CrawlGateError(
+      "Failed to start crawl job",
+      void 0,
+      "CRAWL_START_ERROR"
+    );
+  }
+  return {
+    success: true,
+    id: response.data.id,
+    jobId: response.data.id,
+    status: response.data.status || "scraping",
+    engine: response.data.engine
+  };
+}
+async function getCrawlStatus(http, jobId) {
+  const response = await http.get(`/v1/crawl/${jobId}`);
+  return {
+    id: response.data.id || jobId,
+    status: response.data.status,
+    total: response.data.total || 0,
+    completed: response.data.completed || 0,
+    data: response.data.data || [],
+    engine: response.data.engine,
+    error: response.data.error
+  };
+}
+async function cancelCrawl(http, jobId) {
+  const response = await http.delete(
+    `/v1/crawl/${jobId}`
+  );
+  return response.data.success !== false;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function crawl(http, url, options) {
+  const pollInterval = options?.pollInterval ?? 2e3;
+  const timeout = options?.timeout ?? 300;
+  const { id: jobId } = await startCrawl(http, url, options);
+  const startTime = Date.now();
+  const timeoutMs = timeout * 1e3;
+  while (true) {
+    const status = await getCrawlStatus(http, jobId);
+    if (status.status === "completed") {
+      return status;
+    }
+    if (status.status === "failed") {
+      throw new CrawlGateError(
+        status.error || "Crawl job failed",
+        void 0,
+        "CRAWL_FAILED"
+      );
+    }
+    if (status.status === "cancelled") {
+      throw new CrawlGateError(
+        "Crawl job was cancelled",
+        void 0,
+        "CRAWL_CANCELLED"
+      );
+    }
+    if (Date.now() - startTime > timeoutMs) {
+      throw new JobTimeoutError(jobId, timeout);
+    }
+    await sleep(pollInterval);
+  }
+}
+async function getCrawlErrors(http, jobId) {
+  const response = await http.get(`/v1/crawl/${jobId}/errors`);
+  const payload = response.data.data ?? response.data;
+  return {
+    errors: (payload.errors || []).map((e) => ({
+      id: e.id || "",
+      timestamp: e.timestamp,
+      url: e.url || "",
+      code: e.code,
+      error: e.error || e.message || "Unknown error"
+    })),
+    robotsBlocked: payload.robotsBlocked || []
+  };
+}
+
+// src/methods/map.ts
+function buildMapBody(url, options) {
+  const body = { url };
+  if (options?.engine) {
+    body.engine = options.engine;
+  }
+  if (options?.proxy) {
+    body.proxy = options.proxy;
+  }
+  if (options?.projectId) {
+    body.project_id = options.projectId;
+  }
+  return body;
+}
+async function map(http, url, options) {
+  const body = buildMapBody(url, options);
+  const response = await http.post("/v1/map", body);
+  if (!response.data.success) {
+    throw new CrawlGateError(
+      response.data.error || "Map failed",
+      void 0,
+      "MAP_ERROR"
+    );
+  }
+  return {
+    success: true,
+    links: response.data.links || [],
+    count: response.data.count || response.data.links?.length || 0,
+    engine: response.data.engine
+  };
+}
+
+// src/methods/search.ts
+import { zodToJsonSchema as zodToJsonSchema2 } from "zod-to-json-schema";
+function convertSchema2(schema) {
+  if (schema && typeof schema === "object" && "_def" in schema) {
+    return zodToJsonSchema2(schema);
+  }
+  return schema;
+}
+function buildSearchBody(query, options) {
+  const body = { query };
+  if (options?.limit !== void 0) {
+    body.limit = options.limit;
+  }
+  if (options?.lang) {
+    body.lang = options.lang;
+  }
+  if (options?.country) {
+    body.country = options.country;
+  }
+  if (options?.engines) {
+    body.engines = options.engines;
+  }
+  if (options?.scrapeOptions) {
+    body.scrapeOptions = options.scrapeOptions;
+  }
+  if (options?.engine) {
+    body.engine = options.engine;
+  }
+  if (options?.projectId) {
+    body.project_id = options.projectId;
+  }
+  if (options?.extract) {
+    body.extract = {
+      schema: convertSchema2(options.extract.schema),
+      systemPrompt: options.extract.systemPrompt,
+      provider: options.extract.provider,
+      enableFallback: options.extract.enableFallback
+    };
+    Object.keys(body.extract).forEach((key) => {
+      if (body.extract[key] === void 0) {
+        delete body.extract[key];
+      }
+    });
+  }
+  return body;
+}
+async function search(http, query, options) {
+  const body = buildSearchBody(query, options);
+  const response = await http.post("/v1/search", body);
+  if (!response.data.success) {
+    throw new CrawlGateError(
+      response.data.error || "Search failed",
+      void 0,
+      "SEARCH_ERROR"
+    );
+  }
+  return {
+    success: true,
+    data: response.data.data || [],
+    query: response.data.query || query,
+    totalResults: response.data.totalResults,
+    searchTime: response.data.searchTime,
+    extract: response.data.extract
+  };
+}
+
+// src/methods/batch.ts
+function buildBatchBody(urls, options) {
+  const body = { urls };
+  if (options?.options) {
+    const scrapeOpts = options.options;
+    if (scrapeOpts.engine) body.engine = scrapeOpts.engine;
+    if (scrapeOpts.formats) body.formats = scrapeOpts.formats;
+    if (scrapeOpts.onlyMainContent !== void 0) body.onlyMainContent = scrapeOpts.onlyMainContent;
+    if (scrapeOpts.excludeTags) body.excludeTags = scrapeOpts.excludeTags;
+    if (scrapeOpts.waitFor !== void 0) body.waitFor = scrapeOpts.waitFor;
+    if (scrapeOpts.timeout !== void 0) body.timeout = scrapeOpts.timeout;
+    if (scrapeOpts.proxy) body.proxy = scrapeOpts.proxy;
+  }
+  if (options?.webhook != null) {
+    body.webhook = options.webhook;
+  }
+  if (options?.appendToId != null) {
+    body.appendToId = options.appendToId;
+  }
+  if (options?.ignoreInvalidURLs != null) {
+    body.ignoreInvalidURLs = options.ignoreInvalidURLs;
+  }
+  if (options?.maxConcurrency != null) {
+    body.maxConcurrency = options.maxConcurrency;
+  }
+  if (options?.projectId) {
+    body.project_id = options.projectId;
+  }
+  return body;
+}
+async function startBatchScrape(http, urls, options) {
+  if (!Array.isArray(urls) || urls.length === 0) {
+    throw new CrawlGateError("URLs array cannot be empty", 400, "VALIDATION_ERROR");
+  }
+  const body = buildBatchBody(urls, options);
+  const headers = {};
+  if (options?.idempotencyKey) {
+    headers["Idempotency-Key"] = options.idempotencyKey;
+  }
+  const response = await http.post(
+    "/v1/batch/scrape",
+    body,
+    Object.keys(headers).length > 0 ? headers : void 0
+  );
+  if (!response.data.success && !response.data.id) {
+    throw new CrawlGateError(
+      response.data.error || "Failed to start batch scrape job",
+      void 0,
+      "BATCH_START_ERROR"
+    );
+  }
+  return {
+    success: true,
+    id: response.data.id,
+    url: response.data.url,
+    invalidURLs: response.data.invalidURLs
+  };
+}
+async function getBatchScrapeStatus(http, jobId) {
+  const response = await http.get(
+    `/v1/batch/scrape/${jobId}`
+  );
+  return {
+    id: response.data.id || jobId,
+    status: response.data.status,
+    total: response.data.total || 0,
+    completed: response.data.completed || 0,
+    creditsUsed: response.data.creditsUsed,
+    expiresAt: response.data.expiresAt,
+    next: response.data.next ?? null,
+    data: response.data.data || [],
+    error: response.data.error
+  };
+}
+async function cancelBatchScrape(http, jobId) {
+  const response = await http.delete(
+    `/v1/batch/scrape/${jobId}`
+  );
+  return response.data.status === "cancelled" || response.data.success !== false;
+}
+async function getBatchScrapeErrors(http, jobId) {
+  const response = await http.get(`/v1/batch/scrape/${jobId}/errors`);
+  const payload = response.data.data ?? response.data;
+  return {
+    errors: (payload.errors || []).map((e) => ({
+      id: e.id || "",
+      timestamp: e.timestamp,
+      url: e.url || "",
+      code: e.code,
+      error: e.error || e.message || "Unknown error"
+    })),
+    robotsBlocked: payload.robotsBlocked || []
+  };
+}
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function waitForBatchCompletion(http, jobId, pollInterval = 2e3, timeout) {
+  const startTime = Date.now();
+  const timeoutMs = timeout ? timeout * 1e3 : void 0;
+  while (true) {
+    const status = await getBatchScrapeStatus(http, jobId);
+    if (status.status === "completed") {
+      return status;
+    }
+    if (status.status === "failed") {
+      throw new CrawlGateError(
+        status.error || "Batch scrape job failed",
+        void 0,
+        "BATCH_FAILED"
+      );
+    }
+    if (status.status === "cancelled") {
+      throw new CrawlGateError(
+        "Batch scrape job was cancelled",
+        void 0,
+        "BATCH_CANCELLED"
+      );
+    }
+    if (timeoutMs && Date.now() - startTime > timeoutMs) {
+      throw new JobTimeoutError(jobId, timeout);
+    }
+    await sleep2(Math.max(1e3, pollInterval));
+  }
+}
+async function batchScrape(http, urls, options) {
+  const pollInterval = options?.pollInterval ?? 2e3;
+  const timeout = options?.timeout;
+  const { id: jobId } = await startBatchScrape(http, urls, options);
+  return waitForBatchCompletion(http, jobId, pollInterval, timeout);
+}
+
+// src/methods/extract.ts
+import { zodToJsonSchema as zodToJsonSchema3 } from "zod-to-json-schema";
+function isZodSchema(value) {
+  return value !== null && typeof value === "object" && "_def" in value && (typeof value.safeParse === "function" || typeof value.parse === "function");
+}
+function convertSchema3(schema) {
+  if (isZodSchema(schema)) {
+    return zodToJsonSchema3(schema);
+  }
+  return schema;
+}
+function buildExtractBody(options) {
+  const body = {};
+  if (options.urls) {
+    body.urls = options.urls;
+  }
+  if (options.prompt != null) {
+    body.prompt = options.prompt;
+  }
+  if (options.schema != null) {
+    body.schema = convertSchema3(options.schema);
+  }
+  if (options.systemPrompt != null) {
+    body.systemPrompt = options.systemPrompt;
+  }
+  if (options.allowExternalLinks != null) {
+    body.allowExternalLinks = options.allowExternalLinks;
+  }
+  if (options.enableWebSearch != null) {
+    body.enableWebSearch = options.enableWebSearch;
+  }
+  if (options.showSources != null) {
+    body.showSources = options.showSources;
+  }
+  if (options.ignoreInvalidURLs != null) {
+    body.ignoreInvalidURLs = options.ignoreInvalidURLs;
+  }
+  if (options.provider) {
+    body.provider = options.provider;
+  }
+  if (options.projectId) {
+    body.project_id = options.projectId;
+  }
+  if (options.scrapeOptions) {
+    body.scrapeOptions = options.scrapeOptions;
+  }
+  return body;
+}
+async function startExtract(http, options) {
+  const body = buildExtractBody(options);
+  const response = await http.post("/v1/extract", body);
+  if (response.data.success === false && response.data.error) {
+    throw new CrawlGateError(
+      response.data.error,
+      void 0,
+      "EXTRACT_ERROR"
+    );
+  }
+  return response.data;
+}
+async function getExtractStatus(http, jobId) {
+  const response = await http.get(`/v1/extract/${jobId}`);
+  if (response.data.success === false && response.data.error) {
+    throw new CrawlGateError(
+      response.data.error,
+      void 0,
+      "EXTRACT_STATUS_ERROR"
+    );
+  }
+  return response.data;
+}
+function sleep3(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function waitForExtractCompletion(http, jobId, pollInterval = 2e3, timeout) {
+  const startTime = Date.now();
+  const timeoutMs = timeout ? timeout * 1e3 : void 0;
+  while (true) {
+    const status = await getExtractStatus(http, jobId);
+    if (status.status === "completed") {
+      return status;
+    }
+    if (status.status === "failed") {
+      throw new CrawlGateError(
+        status.error || "Extract job failed",
+        void 0,
+        "EXTRACT_FAILED"
+      );
+    }
+    if (status.status === "cancelled") {
+      throw new CrawlGateError(
+        "Extract job was cancelled",
+        void 0,
+        "EXTRACT_CANCELLED"
+      );
+    }
+    if (timeoutMs && Date.now() - startTime > timeoutMs) {
+      throw new JobTimeoutError(jobId, timeout);
+    }
+    await sleep3(Math.max(1e3, pollInterval));
+  }
+}
+async function extract(http, options) {
+  const pollInterval = options.pollInterval ?? 2e3;
+  const timeout = options.timeout;
+  const started = await startExtract(http, options);
+  if (!started.id) {
+    return started;
+  }
+  if (started.status === "completed") {
+    return started;
+  }
+  return waitForExtractCompletion(http, started.id, pollInterval, timeout);
+}
+
+// src/methods/usage.ts
+async function getConcurrency(http) {
+  const response = await http.get("/v1/concurrency");
+  return {
+    concurrency: response.data.concurrency ?? 0,
+    maxConcurrency: response.data.maxConcurrency ?? 0
+  };
+}
+async function getCreditUsage(http) {
+  const response = await http.get("/v1/credits");
+  return {
+    remainingCredits: response.data.remainingCredits ?? 0,
+    planCredits: response.data.planCredits,
+    billingPeriodStart: response.data.billingPeriodStart,
+    billingPeriodEnd: response.data.billingPeriodEnd
+  };
+}
+async function getTokenUsage(http) {
+  const response = await http.get("/v1/tokens");
+  return {
+    remainingTokens: response.data.remainingTokens ?? 0,
+    planTokens: response.data.planTokens,
+    billingPeriodStart: response.data.billingPeriodStart,
+    billingPeriodEnd: response.data.billingPeriodEnd
+  };
+}
+async function getQueueStatus(http) {
+  const response = await http.get("/v1/queue");
+  return {
+    success: response.data.success ?? true,
+    jobsInQueue: response.data.jobsInQueue ?? 0,
+    activeJobsInQueue: response.data.activeJobsInQueue ?? 0,
+    waitingJobsInQueue: response.data.waitingJobsInQueue ?? 0,
+    maxConcurrency: response.data.maxConcurrency ?? 0,
+    mostRecentSuccess: response.data.mostRecentSuccess
+  };
+}
+
+// src/client.ts
+var CrawlGateClient = class {
+  http;
+  /**
+   * Create a new CrawlGate client
+   *
+   * @param options - Client configuration options
+   * @throws {CrawlGateError} If API key is not provided
+   */
+  constructor(options = {}) {
+    const apiKey = options.apiKey ?? process.env.CRAWLGATE_API_KEY ?? "";
+    const apiUrl = (options.apiUrl ?? process.env.CRAWLGATE_API_URL ?? "https://api.crawlgate.io").replace(/\/$/, "");
+    if (!apiKey) {
+      throw new CrawlGateError(
+        "API key is required. Set CRAWLGATE_API_KEY env variable or pass apiKey option.",
+        void 0,
+        "MISSING_API_KEY"
+      );
+    }
+    this.http = new HttpClient({
+      apiKey,
+      apiUrl,
+      timeoutMs: options.timeoutMs,
+      maxRetries: options.maxRetries,
+      backoffFactor: options.backoffFactor
+    });
+  }
+  // ==========================================================================
+  // Scrape Methods
+  // ==========================================================================
+  /**
+   * Scrape a single URL
+   *
+   * @param url - URL to scrape
+   * @param options - Scrape options
+   * @returns Scraped document with requested formats
+   *
+   * @example
+   * ```typescript
+   * const doc = await client.scrape('https://example.com', {
+   *   engine: 'smart',
+   *   formats: ['markdown', 'html'],
+   *   onlyMainContent: true
+   * });
+   * console.log(doc.markdown);
+   * ```
+   *
+   * @example With LLM extraction
+   * ```typescript
+   * import { z } from 'zod';
+   *
+   * const schema = z.object({
+   *   title: z.string(),
+   *   price: z.number(),
+   *   inStock: z.boolean()
+   * });
+   *
+   * const doc = await client.scrape('https://example.com/product', {
+   *   engine: 'smart',
+   *   extract: {
+   *     schema,
+   *     systemPrompt: 'Extract product details',
+   *     provider: 'openai'
+   *   }
+   * });
+   * console.log(doc.extract?.data);
+   * ```
+   */
+  async scrape(url, options) {
+    return scrape(this.http, url, options);
+  }
+  // ==========================================================================
+  // Batch Scrape Methods
+  // ==========================================================================
+  /**
+   * Start a batch scrape job (async)
+   *
+   * @param urls - Array of URLs to scrape
+   * @param options - Batch scrape options
+   * @returns Batch job ID and initial status
+   *
+   * @example
+   * ```typescript
+   * const { id } = await client.startBatchScrape(
+   *   ['https://a.com', 'https://b.com', 'https://c.com'],
+   *   { options: { formats: ['markdown'] } }
+   * );
+   *
+   * // Poll manually
+   * let status = await client.getBatchScrapeStatus(id);
+   * while (status.status === 'scraping') {
+   *   await new Promise(r => setTimeout(r, 2000));
+   *   status = await client.getBatchScrapeStatus(id);
+   * }
+   * ```
+   */
+  async startBatchScrape(urls, options) {
+    return startBatchScrape(this.http, urls, options);
+  }
+  /**
+   * Get batch scrape job status and data
+   *
+   * @param jobId - Batch job ID
+   * @returns Current job status and scraped data
+   */
+  async getBatchScrapeStatus(jobId) {
+    return getBatchScrapeStatus(this.http, jobId);
+  }
+  /**
+   * Cancel a batch scrape job
+   *
+   * @param jobId - Batch job ID
+   * @returns True if cancelled successfully
+   */
+  async cancelBatchScrape(jobId) {
+    return cancelBatchScrape(this.http, jobId);
+  }
+  /**
+   * Get batch scrape job errors
+   *
+   * @param jobId - Batch job ID
+   * @returns Errors and robots.txt blocked URLs
+   */
+  async getBatchScrapeErrors(jobId) {
+    return getBatchScrapeErrors(this.http, jobId);
+  }
+  /**
+   * Batch scrape multiple URLs and wait for completion
+   *
+   * @param urls - Array of URLs to scrape
+   * @param options - Batch options including pollInterval and timeout
+   * @returns Final job with all scraped data
+   *
+   * @example
+   * ```typescript
+   * const job = await client.batchScrape(
+   *   ['https://a.com', 'https://b.com', 'https://c.com'],
+   *   {
+   *     options: { formats: ['markdown'], engine: 'smart' },
+   *     pollInterval: 2000,
+   *     timeout: 300
+   *   }
+   * );
+   *
+   * console.log(`Scraped ${job.completed} URLs`);
+   * job.data.forEach(doc => console.log(doc.url, doc.markdown?.length));
+   * ```
+   */
+  async batchScrape(urls, options) {
+    return batchScrape(this.http, urls, options);
+  }
+  // ==========================================================================
+  // Crawl Methods
+  // ==========================================================================
+  /**
+   * Start a crawl job (async)
+   *
+   * Use this method when you want to start a crawl and manage polling yourself.
+   * For automatic polling, use the `crawl()` method instead.
+   *
+   * @param url - Root URL to crawl
+   * @param options - Crawl options
+   * @returns Crawl job ID and initial status
+   *
+   * @example
+   * ```typescript
+   * const { id } = await client.startCrawl('https://example.com', {
+   *   limit: 10,
+   *   engine: 'dynamic'
+   * });
+   *
+   * // Poll for status manually
+   * let status = await client.getCrawlStatus(id);
+   * while (status.status === 'scraping') {
+   *   await new Promise(r => setTimeout(r, 2000));
+   *   status = await client.getCrawlStatus(id);
+   * }
+   * ```
+   */
+  async startCrawl(url, options) {
+    return startCrawl(this.http, url, options);
+  }
+  /**
+   * Get crawl job status and data
+   *
+   * @param jobId - Crawl job ID
+   * @returns Current job status and scraped data
+   */
+  async getCrawlStatus(jobId) {
+    return getCrawlStatus(this.http, jobId);
+  }
+  /**
+   * Cancel a crawl job
+   *
+   * @param jobId - Crawl job ID
+   * @returns True if cancelled successfully
+   */
+  async cancelCrawl(jobId) {
+    return cancelCrawl(this.http, jobId);
+  }
+  /**
+   * Get crawl job errors and robots.txt blocks
+   *
+   * @param jobId - Crawl job ID
+   * @returns Errors and robots.txt blocked URLs
+   */
+  async getCrawlErrors(jobId) {
+    return getCrawlErrors(this.http, jobId);
+  }
+  /**
+   * Crawl a website and wait for completion
+   *
+   * This method starts a crawl job and automatically polls until completion.
+   *
+   * @param url - Root URL to crawl
+   * @param options - Crawl options including pollInterval and timeout
+   * @returns Final crawl job with all scraped data
+   *
+   * @example
+   * ```typescript
+   * const job = await client.crawl('https://example.com', {
+   *   limit: 10,
+   *   engine: 'dynamic',
+   *   formats: ['markdown'],
+   *   pollInterval: 2000, // Poll every 2 seconds
+   *   timeout: 300 // 5 minute timeout
+   * });
+   *
+   * console.log(`Crawled ${job.completed} pages`);
+   * job.data.forEach(doc => console.log(doc.url));
+   * ```
+   */
+  async crawl(url, options) {
+    return crawl(this.http, url, options);
+  }
+  // ==========================================================================
+  // Extract Methods (Standalone LLM Extraction)
+  // ==========================================================================
+  /**
+   * Start an extract job (async)
+   *
+   * @param options - Extract request options
+   * @returns Extract job ID or immediate result
+   *
+   * @example
+   * ```typescript
+   * const { id } = await client.startExtract({
+   *   urls: ['https://example.com/product'],
+   *   schema: { name: 'string', price: 'number' },
+   *   provider: 'openai'
+   * });
+   *
+   * // Poll manually
+   * let status = await client.getExtractStatus(id);
+   * while (status.status === 'processing') {
+   *   await new Promise(r => setTimeout(r, 2000));
+   *   status = await client.getExtractStatus(id);
+   * }
+   * console.log(status.data);
+   * ```
+   */
+  async startExtract(options) {
+    return startExtract(this.http, options);
+  }
+  /**
+   * Get extract job status and data
+   *
+   * @param jobId - Extract job ID
+   * @returns Current job status and extracted data
+   */
+  async getExtractStatus(jobId) {
+    return getExtractStatus(this.http, jobId);
+  }
+  /**
+   * Extract structured data from URLs using LLM and wait for completion
+   *
+   * @param options - Extract options including schema, prompt, and timeout
+   * @returns Final extract result with structured data
+   *
+   * @example With Zod schema
+   * ```typescript
+   * import { z } from 'zod';
+   *
+   * const result = await client.extract({
+   *   urls: ['https://example.com/product'],
+   *   schema: z.object({
+   *     name: z.string(),
+   *     price: z.number(),
+   *     inStock: z.boolean(),
+   *     features: z.array(z.string())
+   *   }),
+   *   systemPrompt: 'Extract product information from the page',
+   *   provider: 'openai',
+   *   timeout: 60
+   * });
+   *
+   * console.log(result.data);
+   * ```
+   *
+   * @example With natural language prompt
+   * ```typescript
+   * const result = await client.extract({
+   *   urls: ['https://example.com/about'],
+   *   prompt: 'Extract the company name, founding year, and list of team members',
+   *   enableWebSearch: true
+   * });
+   *
+   * console.log(result.data);
+   * ```
+   */
+  async extract(options) {
+    return extract(this.http, options);
+  }
+  // ==========================================================================
+  // Map Methods
+  // ==========================================================================
+  /**
+   * Map a website to discover all URLs
+   *
+   * @param url - Root URL to map
+   * @param options - Map options
+   * @returns List of discovered URLs
+   *
+   * @example
+   * ```typescript
+   * const result = await client.map('https://example.com', {
+   *   engine: 'dynamic'
+   * });
+   *
+   * console.log(`Found ${result.count} URLs:`);
+   * result.links.forEach(url => console.log(url));
+   * ```
+   */
+  async map(url, options) {
+    return map(this.http, url, options);
+  }
+  // ==========================================================================
+  // Search Methods
+  // ==========================================================================
+  /**
+   * Search the web and optionally scrape results
+   *
+   * @param query - Search query
+   * @param options - Search options
+   * @returns Search results with optional scraped content
+   *
+   * @example Basic search
+   * ```typescript
+   * const results = await client.search('best restaurants in NYC', {
+   *   limit: 10,
+   *   lang: 'en',
+   *   country: 'us'
+   * });
+   *
+   * results.data.forEach(r => {
+   *   console.log(`${r.title}: ${r.url}`);
+   * });
+   * ```
+   *
+   * @example Search with scraping
+   * ```typescript
+   * const results = await client.search('best laptops 2024', {
+   *   limit: 5,
+   *   scrapeOptions: {
+   *     formats: ['markdown']
+   *   },
+   *   engine: 'smart'
+   * });
+   *
+   * results.data.forEach(r => {
+   *   console.log(r.title);
+   *   console.log(r.markdown?.substring(0, 200));
+   * });
+   * ```
+   *
+   * @example Search with LLM extraction
+   * ```typescript
+   * import { z } from 'zod';
+   *
+   * const results = await client.search('iPhone 15 Pro reviews', {
+   *   limit: 5,
+   *   scrapeOptions: { formats: ['markdown'] },
+   *   extract: {
+   *     schema: z.object({
+   *       pros: z.array(z.string()),
+   *       cons: z.array(z.string()),
+   *       rating: z.number()
+   *     }),
+   *     systemPrompt: 'Extract review summary from the content'
+   *   }
+   * });
+   *
+   * console.log(results.extract?.data);
+   * ```
+   */
+  async search(query, options) {
+    return search(this.http, query, options);
+  }
+  // ==========================================================================
+  // Usage & Monitoring Methods
+  // ==========================================================================
+  /**
+   * Get current concurrency usage
+   *
+   * @returns Current and max concurrency
+   *
+   * @example
+   * ```typescript
+   * const { concurrency, maxConcurrency } = await client.getConcurrency();
+   * console.log(`Using ${concurrency}/${maxConcurrency} concurrent requests`);
+   * ```
+   */
+  async getConcurrency() {
+    return getConcurrency(this.http);
+  }
+  /**
+   * Get current credit usage
+   *
+   * @returns Credit usage information
+   *
+   * @example
+   * ```typescript
+   * const credits = await client.getCreditUsage();
+   * console.log(`Remaining credits: ${credits.remainingCredits}`);
+   * ```
+   */
+  async getCreditUsage() {
+    return getCreditUsage(this.http);
+  }
+  /**
+   * Get current token usage (for LLM extraction)
+   *
+   * @returns Token usage information
+   *
+   * @example
+   * ```typescript
+   * const tokens = await client.getTokenUsage();
+   * console.log(`Remaining tokens: ${tokens.remainingTokens}`);
+   * ```
+   */
+  async getTokenUsage() {
+    return getTokenUsage(this.http);
+  }
+  /**
+   * Get queue status information
+   *
+   * @returns Queue status metrics
+   *
+   * @example
+   * ```typescript
+   * const queue = await client.getQueueStatus();
+   * console.log(`Jobs in queue: ${queue.jobsInQueue}`);
+   * console.log(`Active: ${queue.activeJobsInQueue}, Waiting: ${queue.waitingJobsInQueue}`);
+   * ```
+   */
+  async getQueueStatus() {
+    return getQueueStatus(this.http);
+  }
+};
+export {
+  AuthenticationError,
+  CrawlGateClient,
+  CrawlGateError,
+  ExtractionError,
+  JobTimeoutError,
+  RateLimitError,
+  ServiceUnavailableError,
+  ValidationError,
+  CrawlGateClient as default
+};
+//# sourceMappingURL=index.js.map