@crawlgate/sdk 1.0.0 → 1.0.2

package/dist/index.d.cts

@@ -2,18 +2,11 @@ import { ZodTypeAny } from 'zod';
 
 /**
  * Scraping engine selection
- * - `static`: Axios + Cheerio (fast, no JS rendering)
- * - `dynamic`: Playwright headless browser (full JS support)
- * - `smart`: Auto-selects static first, falls back to dynamic if needed
+ * - `static`: Fast HTTP-based scraping (no JavaScript rendering)
+ * - `dynamic`: Full browser scraping with JavaScript support
+ * - `smart`: Auto-selects the best engine based on the target page
  */
 type Engine = "static" | "dynamic" | "smart";
-/**
- * Proxy options
- * - `iproyal`: IPRoyal residential proxy
- * - `tor`: Tor network proxy
- * - `stealth`: Stealth mode with residential proxy (dynamic engine only)
- */
-type ProxyOption = "iproyal" | "tor" | "stealth" | string;
 /**
  * Output format types
  */
@@ -125,10 +118,6 @@ interface ScrapeOptions {
      * Request timeout in milliseconds
      */
     timeout?: number;
-    /**
-     * Proxy configuration
-     */
-    proxy?: ProxyOption;
     /**
      * LLM extraction configuration
      */
@@ -250,10 +239,6 @@ interface CrawlOptions {
      * HTML tags to exclude
      */
     excludeTags?: string[];
-    /**
-     * Proxy configuration
-     */
-    proxy?: ProxyOption;
     /**
      * Project ID for usage tracking
      */
@@ -333,10 +318,6 @@ interface MapOptions {
      * Scraping engine to use
      */
     engine?: Engine;
-    /**
-     * Proxy configuration
-     */
-    proxy?: ProxyOption;
     /**
      * Project ID for usage tracking
      */
@@ -714,90 +695,6 @@ interface ExtractResponse {
      */
     creditsUsed?: number;
 }
-/**
- * Concurrency usage information
- */
-interface ConcurrencyInfo {
-    /**
-     * Current active requests
-     */
-    concurrency: number;
-    /**
-     * Maximum allowed concurrent requests
-     */
-    maxConcurrency: number;
-}
-/**
- * Credit usage information
- */
-interface CreditUsage {
-    /**
-     * Remaining credits
-     */
-    remainingCredits: number;
-    /**
-     * Total plan credits
-     */
-    planCredits?: number;
-    /**
-     * Billing period start date
-     */
-    billingPeriodStart?: string | null;
-    /**
-     * Billing period end date
-     */
-    billingPeriodEnd?: string | null;
-}
-/**
- * Token usage information
- */
-interface TokenUsage {
-    /**
-     * Remaining tokens
-     */
-    remainingTokens: number;
-    /**
-     * Total plan tokens
-     */
-    planTokens?: number;
-    /**
-     * Billing period start date
-     */
-    billingPeriodStart?: string | null;
-    /**
-     * Billing period end date
-     */
-    billingPeriodEnd?: string | null;
-}
-/**
- * Queue status information
- */
-interface QueueStatus {
-    /**
-     * Whether the request was successful
-     */
-    success: boolean;
-    /**
-     * Total jobs in queue
-     */
-    jobsInQueue: number;
-    /**
-     * Active jobs currently processing
-     */
-    activeJobsInQueue: number;
-    /**
-     * Jobs waiting to be processed
-     */
-    waitingJobsInQueue: number;
-    /**
-     * Maximum concurrency
-     */
-    maxConcurrency: number;
-    /**
-     * Timestamp of most recent successful job
-     */
-    mostRecentSuccess: string | null;
-}
 /**
  * Crawl error information
  */
@@ -1231,55 +1128,6 @@ declare class CrawlGateClient {
      * ```
      */
     search(query: string, options?: SearchOptions): Promise<SearchResponse>;
-    /**
-     * Get current concurrency usage
-     *
-     * @returns Current and max concurrency
-     *
-     * @example
-     * ```typescript
-     * const { concurrency, maxConcurrency } = await client.getConcurrency();
-     * console.log(`Using ${concurrency}/${maxConcurrency} concurrent requests`);
-     * ```
-     */
-    getConcurrency(): Promise<ConcurrencyInfo>;
-    /**
-     * Get current credit usage
-     *
-     * @returns Credit usage information
-     *
-     * @example
-     * ```typescript
-     * const credits = await client.getCreditUsage();
-     * console.log(`Remaining credits: ${credits.remainingCredits}`);
-     * ```
-     */
-    getCreditUsage(): Promise<CreditUsage>;
-    /**
-     * Get current token usage (for LLM extraction)
-     *
-     * @returns Token usage information
-     *
-     * @example
-     * ```typescript
-     * const tokens = await client.getTokenUsage();
-     * console.log(`Remaining tokens: ${tokens.remainingTokens}`);
-     * ```
-     */
-    getTokenUsage(): Promise<TokenUsage>;
-    /**
-     * Get queue status information
-     *
-     * @returns Queue status metrics
-     *
-     * @example
-     * ```typescript
-     * const queue = await client.getQueueStatus();
-     * console.log(`Jobs in queue: ${queue.jobsInQueue}`);
-     * console.log(`Active: ${queue.activeJobsInQueue}, Waiting: ${queue.waitingJobsInQueue}`);
-     * ```
-     */
-    getQueueStatus(): Promise<QueueStatus>;
 }
 
 /**
@@ -1353,4 +1201,4 @@ declare class ExtractionError extends CrawlGateError {
     constructor(message: string, provider?: string);
 }
 
-export { AuthenticationError, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse, type ConcurrencyInfo, type CrawlError, type CrawlErrorsResponse, CrawlGateClient, type CrawlGateClientOptions, CrawlGateError, type CrawlJob, type CrawlOptions, type CrawlResponse, type CrawlStatus, type CreditUsage, type Document, type DocumentMetadata, type Engine, type ExtractOptions, type ExtractRequestOptions, type ExtractResponse, type ExtractResult, type ExtractStatus, ExtractionError, type FormatType, JobTimeoutError, type JsonSchema, type LLMProvider, type MapOptions, type MapResponse, type PaginationConfig, type ProxyOption, type QueueStatus, RateLimitError, type ScrapeOptions, type ScrapeResponse, type SearchOptions, type SearchResponse, type SearchResult, ServiceUnavailableError, type TokenUsage, ValidationError, type WebhookConfig, CrawlGateClient as default };
+export { AuthenticationError, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse, type CrawlError, type CrawlErrorsResponse, CrawlGateClient, type CrawlGateClientOptions, CrawlGateError, type CrawlJob, type CrawlOptions, type CrawlResponse, type CrawlStatus, type Document, type DocumentMetadata, type Engine, type ExtractOptions, type ExtractRequestOptions, type ExtractResponse, type ExtractResult, type ExtractStatus, ExtractionError, type FormatType, JobTimeoutError, type JsonSchema, type LLMProvider, type MapOptions, type MapResponse, type PaginationConfig, RateLimitError, type ScrapeOptions, type ScrapeResponse, type SearchOptions, type SearchResponse, type SearchResult, ServiceUnavailableError, ValidationError, type WebhookConfig, CrawlGateClient as default };

package/dist/index.d.ts

@@ -2,18 +2,11 @@ import { ZodTypeAny } from 'zod';
 
 /**
  * Scraping engine selection
- * - `static`: Axios + Cheerio (fast, no JS rendering)
- * - `dynamic`: Playwright headless browser (full JS support)
- * - `smart`: Auto-selects static first, falls back to dynamic if needed
+ * - `static`: Fast HTTP-based scraping (no JavaScript rendering)
+ * - `dynamic`: Full browser scraping with JavaScript support
+ * - `smart`: Auto-selects the best engine based on the target page
  */
 type Engine = "static" | "dynamic" | "smart";
-/**
- * Proxy options
- * - `iproyal`: IPRoyal residential proxy
- * - `tor`: Tor network proxy
- * - `stealth`: Stealth mode with residential proxy (dynamic engine only)
- */
-type ProxyOption = "iproyal" | "tor" | "stealth" | string;
 /**
  * Output format types
  */
@@ -125,10 +118,6 @@ interface ScrapeOptions {
      * Request timeout in milliseconds
      */
     timeout?: number;
-    /**
-     * Proxy configuration
-     */
-    proxy?: ProxyOption;
     /**
      * LLM extraction configuration
      */
@@ -250,10 +239,6 @@ interface CrawlOptions {
      * HTML tags to exclude
      */
     excludeTags?: string[];
-    /**
-     * Proxy configuration
-     */
-    proxy?: ProxyOption;
     /**
      * Project ID for usage tracking
      */
@@ -333,10 +318,6 @@ interface MapOptions {
      * Scraping engine to use
      */
     engine?: Engine;
-    /**
-     * Proxy configuration
-     */
-    proxy?: ProxyOption;
     /**
      * Project ID for usage tracking
      */
@@ -714,90 +695,6 @@ interface ExtractResponse {
      */
     creditsUsed?: number;
 }
-/**
- * Concurrency usage information
- */
-interface ConcurrencyInfo {
-    /**
-     * Current active requests
-     */
-    concurrency: number;
-    /**
-     * Maximum allowed concurrent requests
-     */
-    maxConcurrency: number;
-}
-/**
- * Credit usage information
- */
-interface CreditUsage {
-    /**
-     * Remaining credits
-     */
-    remainingCredits: number;
-    /**
-     * Total plan credits
-     */
-    planCredits?: number;
-    /**
-     * Billing period start date
-     */
-    billingPeriodStart?: string | null;
-    /**
-     * Billing period end date
-     */
-    billingPeriodEnd?: string | null;
-}
-/**
- * Token usage information
- */
-interface TokenUsage {
-    /**
-     * Remaining tokens
-     */
-    remainingTokens: number;
-    /**
-     * Total plan tokens
-     */
-    planTokens?: number;
-    /**
-     * Billing period start date
-     */
-    billingPeriodStart?: string | null;
-    /**
-     * Billing period end date
-     */
-    billingPeriodEnd?: string | null;
-}
-/**
- * Queue status information
- */
-interface QueueStatus {
-    /**
-     * Whether the request was successful
-     */
-    success: boolean;
-    /**
-     * Total jobs in queue
-     */
-    jobsInQueue: number;
-    /**
-     * Active jobs currently processing
-     */
-    activeJobsInQueue: number;
-    /**
-     * Jobs waiting to be processed
-     */
-    waitingJobsInQueue: number;
-    /**
-     * Maximum concurrency
-     */
-    maxConcurrency: number;
-    /**
-     * Timestamp of most recent successful job
-     */
-    mostRecentSuccess: string | null;
-}
 /**
  * Crawl error information
  */
@@ -1231,55 +1128,6 @@ declare class CrawlGateClient {
      * ```
      */
     search(query: string, options?: SearchOptions): Promise<SearchResponse>;
-    /**
-     * Get current concurrency usage
-     *
-     * @returns Current and max concurrency
-     *
-     * @example
-     * ```typescript
-     * const { concurrency, maxConcurrency } = await client.getConcurrency();
-     * console.log(`Using ${concurrency}/${maxConcurrency} concurrent requests`);
-     * ```
-     */
-    getConcurrency(): Promise<ConcurrencyInfo>;
-    /**
-     * Get current credit usage
-     *
-     * @returns Credit usage information
-     *
-     * @example
-     * ```typescript
-     * const credits = await client.getCreditUsage();
-     * console.log(`Remaining credits: ${credits.remainingCredits}`);
-     * ```
-     */
-    getCreditUsage(): Promise<CreditUsage>;
-    /**
-     * Get current token usage (for LLM extraction)
-     *
-     * @returns Token usage information
-     *
-     * @example
-     * ```typescript
-     * const tokens = await client.getTokenUsage();
-     * console.log(`Remaining tokens: ${tokens.remainingTokens}`);
-     * ```
-     */
-    getTokenUsage(): Promise<TokenUsage>;
-    /**
-     * Get queue status information
-     *
-     * @returns Queue status metrics
-     *
-     * @example
-     * ```typescript
-     * const queue = await client.getQueueStatus();
-     * console.log(`Jobs in queue: ${queue.jobsInQueue}`);
-     * console.log(`Active: ${queue.activeJobsInQueue}, Waiting: ${queue.waitingJobsInQueue}`);
-     * ```
-     */
-    getQueueStatus(): Promise<QueueStatus>;
 }
 
 /**
@@ -1353,4 +1201,4 @@ declare class ExtractionError extends CrawlGateError {
     constructor(message: string, provider?: string);
 }
 
-export { AuthenticationError, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse, type ConcurrencyInfo, type CrawlError, type CrawlErrorsResponse, CrawlGateClient, type CrawlGateClientOptions, CrawlGateError, type CrawlJob, type CrawlOptions, type CrawlResponse, type CrawlStatus, type CreditUsage, type Document, type DocumentMetadata, type Engine, type ExtractOptions, type ExtractRequestOptions, type ExtractResponse, type ExtractResult, type ExtractStatus, ExtractionError, type FormatType, JobTimeoutError, type JsonSchema, type LLMProvider, type MapOptions, type MapResponse, type PaginationConfig, type ProxyOption, type QueueStatus, RateLimitError, type ScrapeOptions, type ScrapeResponse, type SearchOptions, type SearchResponse, type SearchResult, ServiceUnavailableError, type TokenUsage, ValidationError, type WebhookConfig, CrawlGateClient as default };
+export { AuthenticationError, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse, type CrawlError, type CrawlErrorsResponse, CrawlGateClient, type CrawlGateClientOptions, CrawlGateError, type CrawlJob, type CrawlOptions, type CrawlResponse, type CrawlStatus, type Document, type DocumentMetadata, type Engine, type ExtractOptions, type ExtractRequestOptions, type ExtractResponse, type ExtractResult, type ExtractStatus, ExtractionError, type FormatType, JobTimeoutError, type JsonSchema, type LLMProvider, type MapOptions, type MapResponse, type PaginationConfig, RateLimitError, type ScrapeOptions, type ScrapeResponse, type SearchOptions, type SearchResponse, type SearchResult, ServiceUnavailableError, ValidationError, type WebhookConfig, CrawlGateClient as default };

package/dist/index.js

@@ -248,9 +248,6 @@ function buildScrapeBody(url, options) {
   if (options?.timeout !== void 0) {
     body.timeout = options.timeout;
   }
-  if (options?.proxy) {
-    body.proxy = options.proxy;
-  }
   if (options?.projectId) {
     body.project_id = options.projectId;
   }
@@ -272,20 +269,21 @@ function buildScrapeBody(url, options) {
 async function scrape(http, url, options) {
   const body = buildScrapeBody(url, options);
   const response = await http.post("/v1/scrape", body);
-  if (!response.data.success) {
+  const data = response.data;
+  if (data.success === false) {
     throw new CrawlGateError(
-      response.data.error || "Scrape failed",
+      data.error || "Scrape failed",
       void 0,
-      "SCRAPE_ERROR"
+      data.code || "SCRAPE_ERROR"
     );
   }
-  if (!response.data.data) {
-    throw new CrawlGateError("No data returned from scrape", void 0, "NO_DATA");
+  if (data.success === true && data.data) {
+    return data.data;
   }
-  const document = {
-    ...response.data.data
-  };
-  return document;
+  if (data.url || data.markdown || data.html) {
+    return data;
+  }
+  throw new CrawlGateError("No data returned from scrape", void 0, "NO_DATA");
 }
 
 // src/methods/crawl.ts
@@ -306,9 +304,6 @@ function buildCrawlBody(url, options) {
   if (options?.excludeTags) {
     body.excludeTags = options.excludeTags;
   }
-  if (options?.proxy) {
-    body.proxy = options.proxy;
-  }
   if (options?.projectId) {
     body.project_id = options.projectId;
   }
@@ -405,9 +400,6 @@ function buildMapBody(url, options) {
   if (options?.engine) {
     body.engine = options.engine;
   }
-  if (options?.proxy) {
-    body.proxy = options.proxy;
-  }
   if (options?.projectId) {
     body.project_id = options.projectId;
   }
@@ -508,7 +500,6 @@ function buildBatchBody(urls, options) {
     if (scrapeOpts.excludeTags) body.excludeTags = scrapeOpts.excludeTags;
     if (scrapeOpts.waitFor !== void 0) body.waitFor = scrapeOpts.waitFor;
     if (scrapeOpts.timeout !== void 0) body.timeout = scrapeOpts.timeout;
-    if (scrapeOpts.proxy) body.proxy = scrapeOpts.proxy;
   }
   if (options?.webhook != null) {
     body.webhook = options.webhook;
@@ -744,44 +735,6 @@ async function extract(http, options) {
   return waitForExtractCompletion(http, started.id, pollInterval, timeout);
 }
 
-// src/methods/usage.ts
-async function getConcurrency(http) {
-  const response = await http.get("/v1/concurrency");
-  return {
-    concurrency: response.data.concurrency ?? 0,
-    maxConcurrency: response.data.maxConcurrency ?? 0
-  };
-}
-async function getCreditUsage(http) {
-  const response = await http.get("/v1/credits");
-  return {
-    remainingCredits: response.data.remainingCredits ?? 0,
-    planCredits: response.data.planCredits,
-    billingPeriodStart: response.data.billingPeriodStart,
-    billingPeriodEnd: response.data.billingPeriodEnd
-  };
-}
-async function getTokenUsage(http) {
-  const response = await http.get("/v1/tokens");
-  return {
-    remainingTokens: response.data.remainingTokens ?? 0,
-    planTokens: response.data.planTokens,
-    billingPeriodStart: response.data.billingPeriodStart,
-    billingPeriodEnd: response.data.billingPeriodEnd
-  };
-}
-async function getQueueStatus(http) {
-  const response = await http.get("/v1/queue");
-  return {
-    success: response.data.success ?? true,
-    jobsInQueue: response.data.jobsInQueue ?? 0,
-    activeJobsInQueue: response.data.activeJobsInQueue ?? 0,
-    waitingJobsInQueue: response.data.waitingJobsInQueue ?? 0,
-    maxConcurrency: response.data.maxConcurrency ?? 0,
-    mostRecentSuccess: response.data.mostRecentSuccess
-  };
-}
-
 // src/client.ts
 var CrawlGateClient = class {
   http;
@@ -1180,66 +1133,6 @@ var CrawlGateClient = class {
   async search(query, options) {
     return search(this.http, query, options);
   }
-  // ==========================================================================
-  // Usage & Monitoring Methods
-  // ==========================================================================
-  /**
-   * Get current concurrency usage
-   *
-   * @returns Current and max concurrency
-   *
-   * @example
-   * ```typescript
-   * const { concurrency, maxConcurrency } = await client.getConcurrency();
-   * console.log(`Using ${concurrency}/${maxConcurrency} concurrent requests`);
-   * ```
-   */
-  async getConcurrency() {
-    return getConcurrency(this.http);
-  }
-  /**
-   * Get current credit usage
-   *
-   * @returns Credit usage information
-   *
-   * @example
-   * ```typescript
-   * const credits = await client.getCreditUsage();
-   * console.log(`Remaining credits: ${credits.remainingCredits}`);
-   * ```
-   */
-  async getCreditUsage() {
-    return getCreditUsage(this.http);
-  }
-  /**
-   * Get current token usage (for LLM extraction)
-   *
-   * @returns Token usage information
-   *
-   * @example
-   * ```typescript
-   * const tokens = await client.getTokenUsage();
-   * console.log(`Remaining tokens: ${tokens.remainingTokens}`);
-   * ```
-   */
-  async getTokenUsage() {
-    return getTokenUsage(this.http);
-  }
-  /**
-   * Get queue status information
-   *
-   * @returns Queue status metrics
-   *
-   * @example
-   * ```typescript
-   * const queue = await client.getQueueStatus();
-   * console.log(`Jobs in queue: ${queue.jobsInQueue}`);
-   * console.log(`Active: ${queue.activeJobsInQueue}, Waiting: ${queue.waitingJobsInQueue}`);
-   * ```
-   */
-  async getQueueStatus() {
-    return getQueueStatus(this.http);
-  }
 };
 export {
   AuthenticationError,