@crawlgate/sdk 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,1356 @@
+import { ZodTypeAny } from 'zod';
+
+/**
+ * Scraping engine selection
+ * - `static`: Axios + Cheerio (fast, no JS rendering)
+ * - `dynamic`: Playwright headless browser (full JS support)
+ * - `smart`: Auto-selects static first, falls back to dynamic if needed
+ */
+type Engine = "static" | "dynamic" | "smart";
+/**
+ * Proxy options
+ * - `iproyal`: IPRoyal residential proxy
+ * - `tor`: Tor network proxy
+ * - `stealth`: Stealth mode with residential proxy (dynamic engine only)
+ */
+type ProxyOption = "iproyal" | "tor" | "stealth" | string;
+/**
+ * Output format types
+ */
+type FormatType = "markdown" | "html" | "rawHtml" | "text";
+/**
+ * LLM provider for extraction
+ */
+type LLMProvider = "openai" | "anthropic";
+/**
+ * Configuration options for CrawlGateClient
+ */
+interface CrawlGateClientOptions {
+    /**
+     * API key for authentication (falls back to CRAWLGATE_API_KEY env variable)
+     */
+    apiKey?: string;
+    /**
+     * Base URL for the API (default: https://api.crawlgate.io)
+     */
+    apiUrl?: string;
+    /**
+     * Request timeout in milliseconds (default: 90000)
+     */
+    timeoutMs?: number;
+    /**
+     * Maximum number of retries for failed requests (default: 3)
+     */
+    maxRetries?: number;
+    /**
+     * Backoff factor for retries in seconds (default: 0.5)
+     */
+    backoffFactor?: number;
+}
+/**
+ * JSON Schema definition for LLM extraction
+ */
+type JsonSchema = Record<string, unknown> | ZodTypeAny;
+/**
+ * Options for LLM-powered data extraction
+ */
+interface ExtractOptions {
+    /**
+     * JSON Schema or Zod schema defining the structure of extracted data
+     */
+    schema: JsonSchema;
+    /**
+     * System prompt to guide the LLM extraction
+     */
+    systemPrompt?: string;
+    /**
+     * LLM provider to use (default: openai)
+     */
+    provider?: LLMProvider;
+    /**
+     * Enable fallback to alternative provider if primary fails
+     */
+    enableFallback?: boolean;
+}
+/**
+ * Result of LLM extraction
+ */
+interface ExtractResult {
+    /**
+     * Extracted structured data matching the schema
+     */
+    data: unknown;
+    /**
+     * Provider used for extraction
+     */
+    provider: string;
+    /**
+     * Model used for extraction
+     */
+    model: string;
+    /**
+     * Token usage statistics
+     */
+    usage?: {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+    };
+}
+/**
+ * Options for scraping a single URL
+ */
+interface ScrapeOptions {
+    /**
+     * Scraping engine to use
+     */
+    engine?: Engine;
+    /**
+     * Output formats to return
+     */
+    formats?: FormatType[];
+    /**
+     * Extract only the main content (removes headers, footers, sidebars)
+     */
+    onlyMainContent?: boolean;
+    /**
+     * HTML tags to exclude from output
+     */
+    excludeTags?: string[];
+    /**
+     * Wait for specified milliseconds before scraping (dynamic engine)
+     */
+    waitFor?: number;
+    /**
+     * Request timeout in milliseconds
+     */
+    timeout?: number;
+    /**
+     * Proxy configuration
+     */
+    proxy?: ProxyOption;
+    /**
+     * LLM extraction configuration
+     */
+    extract?: ExtractOptions;
+    /**
+     * Project ID for usage tracking
+     */
+    projectId?: string;
+}
+/**
+ * Metadata from scraped page
+ */
+interface DocumentMetadata {
+    title?: string;
+    description?: string;
+    language?: string;
+    sourceURL?: string;
+    ogTitle?: string;
+    ogDescription?: string;
+    ogImage?: string;
+    favicon?: string;
+    [key: string]: unknown;
+}
+/**
+ * Scraped document data
+ */
+interface Document {
+    /**
+     * Source URL
+     */
+    url: string;
+    /**
+     * Content in Markdown format
+     */
+    markdown?: string;
+    /**
+     * Content in HTML format
+     */
+    html?: string;
+    /**
+     * Raw HTML content
+     */
+    rawHtml?: string;
+    /**
+     * Plain text content
+     */
+    text?: string;
+    /**
+     * Page metadata
+     */
+    metadata?: DocumentMetadata;
+    /**
+     * Scrape duration in milliseconds
+     */
+    scrapeTime?: number;
+    /**
+     * LLM extraction result (if extract option was provided)
+     */
+    extract?: ExtractResult;
+    /**
+     * Alias for extract.data (backward compatibility)
+     */
+    json?: unknown;
+}
+/**
+ * Response from scrape endpoint
+ */
+interface ScrapeResponse {
+    /**
+     * Whether the request was successful
+     */
+    success: boolean;
+    /**
+     * Scraped document data
+     */
+    data?: Document;
+    /**
+     * Engine that was requested
+     */
+    engine?: Engine;
+    /**
+     * Engine that was actually used (for smart engine)
+     */
+    engineUsed?: "static" | "dynamic";
+    /**
+     * Request ID for tracking
+     */
+    requestId?: string;
+    /**
+     * Error message if failed
+     */
+    error?: string;
+}
+/**
+ * Crawl job status
+ */
+type CrawlStatus = "scraping" | "completed" | "failed" | "cancelled";
+/**
+ * Options for crawling a website
+ */
+interface CrawlOptions {
+    /**
+     * Scraping engine to use
+     */
+    engine?: Engine;
+    /**
+     * Maximum number of pages to crawl
+     */
+    limit?: number;
+    /**
+     * Output formats to return
+     */
+    formats?: FormatType[];
+    /**
+     * Extract only the main content
+     */
+    onlyMainContent?: boolean;
+    /**
+     * HTML tags to exclude
+     */
+    excludeTags?: string[];
+    /**
+     * Proxy configuration
+     */
+    proxy?: ProxyOption;
+    /**
+     * Project ID for usage tracking
+     */
+    projectId?: string;
+    /**
+     * Poll interval in milliseconds (for waiter method)
+     */
+    pollInterval?: number;
+    /**
+     * Maximum wait time in seconds (for waiter method)
+     */
+    timeout?: number;
+}
+/**
+ * Response from starting a crawl job
+ */
+interface CrawlResponse {
+    /**
+     * Whether the request was successful
+     */
+    success: boolean;
+    /**
+     * Crawl job ID
+     */
+    id: string;
+    /**
+     * Alias for id
+     */
+    jobId?: string;
+    /**
+     * Initial job status
+     */
+    status: CrawlStatus;
+    /**
+     * Engine being used
+     */
+    engine?: string;
+}
+/**
+ * Crawl job status and data
+ */
+interface CrawlJob {
+    /**
+     * Job ID
+     */
+    id: string;
+    /**
+     * Current status
+     */
+    status: CrawlStatus;
+    /**
+     * Total pages to crawl
+     */
+    total: number;
+    /**
+     * Pages completed
+     */
+    completed: number;
+    /**
+     * Scraped documents (empty until completed)
+     */
+    data: Document[];
+    /**
+     * Engine being used
+     */
+    engine?: string;
+    /**
+     * Error message if failed
+     */
+    error?: string;
+}
+/**
+ * Options for mapping URLs on a website
+ */
+interface MapOptions {
+    /**
+     * Scraping engine to use
+     */
+    engine?: Engine;
+    /**
+     * Proxy configuration
+     */
+    proxy?: ProxyOption;
+    /**
+     * Project ID for usage tracking
+     */
+    projectId?: string;
+}
+/**
+ * Response from map endpoint
+ */
+interface MapResponse {
+    /**
+     * Whether the request was successful
+     */
+    success: boolean;
+    /**
+     * Discovered URLs
+     */
+    links: string[];
+    /**
+     * Number of URLs found
+     */
+    count: number;
+    /**
+     * Engine used
+     */
+    engine?: string;
+    /**
+     * Error message if failed
+     */
+    error?: string;
+}
+/**
+ * Options for web search
+ */
+interface SearchOptions {
+    /**
+     * Maximum number of results (default: 10)
+     */
+    limit?: number;
+    /**
+     * Language code (default: en)
+     */
+    lang?: string;
+    /**
+     * Country code (default: us)
+     */
+    country?: string;
+    /**
+     * Search engines to use (default: all available)
+     */
+    engines?: string[];
+    /**
+     * Scrape options (if provided, results will be scraped)
+     */
+    scrapeOptions?: {
+        formats?: FormatType[];
+    };
+    /**
+     * Scraping engine for scraping results
+     */
+    engine?: Engine;
+    /**
+     * LLM extraction on combined search results
+     */
+    extract?: ExtractOptions;
+    /**
+     * Project ID for usage tracking
+     */
+    projectId?: string;
+}
+/**
+ * Individual search result
+ */
+interface SearchResult {
+    /**
+     * Result URL
+     */
+    url: string;
+    /**
+     * Result title
+     */
+    title?: string;
+    /**
+     * Result description/snippet
+     */
+    description?: string;
+    /**
+     * Search engine that returned this result
+     */
+    engine?: string;
+    /**
+     * Relevance score
+     */
+    score?: number;
+    /**
+     * Position in search results
+     */
+    position?: number;
+    /**
+     * Scraped markdown content (if scrapeOptions provided)
+     */
+    markdown?: string | null;
+    /**
+     * Scraped HTML content (if scrapeOptions provided)
+     */
+    html?: string | null;
+    /**
+     * Page metadata (if scraped)
+     */
+    metadata?: DocumentMetadata;
+    /**
+     * Whether scraping was successful
+     */
+    scrapeSuccess?: boolean;
+}
+/**
+ * Response from search endpoint
+ */
+interface SearchResponse {
+    /**
+     * Whether the request was successful
+     */
+    success: boolean;
+    /**
+     * Search results
+     */
+    data: SearchResult[];
+    /**
+     * Original search query
+     */
+    query: string;
+    /**
+     * Total results available
+     */
+    totalResults?: number;
+    /**
+     * Search/scrape time in milliseconds
+     */
+    searchTime?: number;
+    /**
+     * LLM extraction result (if extract option provided)
+     */
+    extract?: ExtractResult;
+    /**
+     * Error message if failed
+     */
+    error?: string;
+}
+/**
+ * Options for batch scraping multiple URLs
+ */
+interface BatchScrapeOptions {
+    /**
+     * Scrape options to apply to all URLs
+     */
+    options?: ScrapeOptions;
+    /**
+     * Webhook URL for job status updates
+     */
+    webhook?: string | WebhookConfig;
+    /**
+     * Append results to existing batch job
+     */
+    appendToId?: string;
+    /**
+     * Skip invalid URLs instead of failing
+     */
+    ignoreInvalidURLs?: boolean;
+    /**
+     * Maximum concurrent scrapes
+     */
+    maxConcurrency?: number;
+    /**
+     * Idempotency key for deduplication
+     */
+    idempotencyKey?: string;
+    /**
+     * Project ID for usage tracking
+     */
+    projectId?: string;
+    /**
+     * Poll interval in milliseconds (for waiter method)
+     */
+    pollInterval?: number;
+    /**
+     * Maximum wait time in seconds (for waiter method)
+     */
+    timeout?: number;
+}
+/**
+ * Webhook configuration
+ */
+interface WebhookConfig {
+    /**
+     * Webhook URL
+     */
+    url: string;
+    /**
+     * Custom headers to send with webhook
+     */
+    headers?: Record<string, string>;
+    /**
+     * Metadata to include in webhook payload
+     */
+    metadata?: Record<string, string>;
+    /**
+     * Events to trigger webhook
+     */
+    events?: Array<"completed" | "failed" | "page" | "started">;
+}
+/**
+ * Response from starting a batch scrape job
+ */
+interface BatchScrapeResponse {
+    /**
+     * Whether the request was successful
+     */
+    success: boolean;
+    /**
+     * Batch job ID
+     */
+    id: string;
+    /**
+     * URL to check job status
+     */
+    url?: string;
+    /**
+     * Invalid URLs that were skipped
+     */
+    invalidURLs?: string[];
+    /**
+     * Error message if failed
+     */
+    error?: string;
+}
+/**
+ * Batch scrape job status and data
+ */
+interface BatchScrapeJob {
+    /**
+     * Job ID
+     */
+    id: string;
+    /**
+     * Current status
+     */
+    status: CrawlStatus;
+    /**
+     * Total URLs to scrape
+     */
+    total: number;
+    /**
+     * URLs completed
+     */
+    completed: number;
+    /**
+     * Credits used
+     */
+    creditsUsed?: number;
+    /**
+     * Job expiration time
+     */
+    expiresAt?: string;
+    /**
+     * Next page URL for pagination
+     */
+    next?: string | null;
+    /**
+     * Scraped documents
+     */
+    data: Document[];
+    /**
+     * Error message if failed
+     */
+    error?: string;
+}
+/**
+ * Options for standalone LLM extraction
+ */
+interface ExtractRequestOptions {
+    /**
+     * URLs to extract data from
+     */
+    urls?: string[];
+    /**
+     * Natural language prompt for extraction
+     */
+    prompt?: string;
+    /**
+     * JSON Schema or Zod schema for structured extraction
+     */
+    schema?: JsonSchema;
+    /**
+     * System prompt for LLM
+     */
+    systemPrompt?: string;
+    /**
+     * Allow following external links
+     */
+    allowExternalLinks?: boolean;
+    /**
+     * Enable web search for additional context
+     */
+    enableWebSearch?: boolean;
+    /**
+     * Include source URLs in response
+     */
+    showSources?: boolean;
+    /**
+     * Scrape options for URLs
+     */
+    scrapeOptions?: ScrapeOptions;
+    /**
+     * Skip invalid URLs instead of failing
+     */
+    ignoreInvalidURLs?: boolean;
+    /**
+     * LLM provider to use
+     */
+    provider?: LLMProvider;
+    /**
+     * Project ID for usage tracking
+     */
+    projectId?: string;
+    /**
+     * Poll interval in milliseconds (for waiter method)
+     */
+    pollInterval?: number;
+    /**
+     * Maximum wait time in seconds (for waiter method)
+     */
+    timeout?: number;
+}
+/**
+ * Extract job status
+ */
+type ExtractStatus = "processing" | "completed" | "failed" | "cancelled";
+/**
+ * Response from extract endpoint
+ */
+interface ExtractResponse {
+    /**
+     * Whether the request was successful
+     */
+    success?: boolean;
+    /**
+     * Extract job ID (for async operations)
+     */
+    id?: string;
+    /**
+     * Job status
+     */
+    status?: ExtractStatus;
+    /**
+     * Extracted data
+     */
+    data?: unknown;
+    /**
+     * Error message if failed
+     */
+    error?: string;
+    /**
+     * Warning message
+     */
+    warning?: string;
+    /**
+     * Source URLs used for extraction
+     */
+    sources?: Record<string, unknown>;
+    /**
+     * Job expiration time
+     */
+    expiresAt?: string;
+    /**
+     * Credits used
+     */
+    creditsUsed?: number;
+}
+/**
+ * Concurrency usage information
+ */
+interface ConcurrencyInfo {
+    /**
+     * Current active requests
+     */
+    concurrency: number;
+    /**
+     * Maximum allowed concurrent requests
+     */
+    maxConcurrency: number;
+}
+/**
+ * Credit usage information
+ */
+interface CreditUsage {
+    /**
+     * Remaining credits
+     */
+    remainingCredits: number;
+    /**
+     * Total plan credits
+     */
+    planCredits?: number;
+    /**
+     * Billing period start date
+     */
+    billingPeriodStart?: string | null;
+    /**
+     * Billing period end date
+     */
+    billingPeriodEnd?: string | null;
+}
+/**
+ * Token usage information
+ */
+interface TokenUsage {
+    /**
+     * Remaining tokens
+     */
+    remainingTokens: number;
+    /**
+     * Total plan tokens
+     */
+    planTokens?: number;
+    /**
+     * Billing period start date
+     */
+    billingPeriodStart?: string | null;
+    /**
+     * Billing period end date
+     */
+    billingPeriodEnd?: string | null;
+}
+/**
+ * Queue status information
+ */
+interface QueueStatus {
+    /**
+     * Whether the request was successful
+     */
+    success: boolean;
+    /**
+     * Total jobs in queue
+     */
+    jobsInQueue: number;
+    /**
+     * Active jobs currently processing
+     */
+    activeJobsInQueue: number;
+    /**
+     * Jobs waiting to be processed
+     */
+    waitingJobsInQueue: number;
+    /**
+     * Maximum concurrency
+     */
+    maxConcurrency: number;
+    /**
+     * Timestamp of most recent successful job
+     */
+    mostRecentSuccess: string | null;
+}
+/**
+ * Crawl error information
+ */
+interface CrawlError {
+    /**
+     * Error ID
+     */
+    id: string;
+    /**
+     * Timestamp of error
+     */
+    timestamp?: string;
+    /**
+     * URL that caused the error
+     */
+    url: string;
+    /**
+     * Error code
+     */
+    code?: string;
+    /**
+     * Error message
+     */
+    error: string;
+}
+/**
+ * Response from crawl errors endpoint
+ */
+interface CrawlErrorsResponse {
+    /**
+     * List of errors encountered
+     */
+    errors: CrawlError[];
+    /**
+     * URLs blocked by robots.txt
+     */
+    robotsBlocked: string[];
+}
+/**
+ * Pagination configuration for large result sets
+ */
+interface PaginationConfig {
+    /**
+     * Automatically fetch all pages (default: true)
+     */
+    autoPaginate?: boolean;
+    /**
+     * Maximum number of pages to fetch
+     */
+    maxPages?: number;
+    /**
+     * Maximum total results to return
+     */
+    maxResults?: number;
+    /**
+     * Maximum time to spend fetching pages (seconds)
+     */
+    maxWaitTime?: number;
+}
+
+/**
+ * CrawlGate SDK Client
+ *
+ * @example
+ * ```typescript
+ * import { CrawlGateClient } from '@crawlgate/sdk';
+ *
+ * const client = new CrawlGateClient({
+ *   apiKey: 'sk_live_...',
+ *   apiUrl: 'https://api.crawlgate.io'
+ * });
+ *
+ * // Scrape a single URL
+ * const doc = await client.scrape('https://example.com', {
+ *   engine: 'smart',
+ *   formats: ['markdown', 'html']
+ * });
+ *
+ * // Batch scrape multiple URLs
+ * const job = await client.batchScrape(['https://a.com', 'https://b.com'], {
+ *   options: { formats: ['markdown'] }
+ * });
+ *
+ * // Crawl a website
+ * const crawlJob = await client.crawl('https://example.com', {
+ *   limit: 10,
+ *   engine: 'dynamic'
+ * });
+ *
+ * // Extract structured data with LLM
+ * const extracted = await client.extract({
+ *   urls: ['https://example.com/product'],
+ *   schema: { name: 'string', price: 'number' },
+ *   provider: 'openai'
+ * });
+ *
+ * // Search the web
+ * const results = await client.search('best restaurants', {
+ *   limit: 5,
+ *   scrapeOptions: { formats: ['markdown'] }
+ * });
+ * ```
+ */
+declare class CrawlGateClient {
+    private readonly http;
+    /**
+     * Create a new CrawlGate client
+     *
+     * @param options - Client configuration options
+     * @throws {CrawlGateError} If API key is not provided
+     */
+    constructor(options?: CrawlGateClientOptions);
+    /**
+     * Scrape a single URL
+     *
+     * @param url - URL to scrape
+     * @param options - Scrape options
+     * @returns Scraped document with requested formats
+     *
+     * @example
+     * ```typescript
+     * const doc = await client.scrape('https://example.com', {
+     *   engine: 'smart',
+     *   formats: ['markdown', 'html'],
+     *   onlyMainContent: true
+     * });
+     * console.log(doc.markdown);
+     * ```
+     *
+     * @example With LLM extraction
+     * ```typescript
+     * import { z } from 'zod';
+     *
+     * const schema = z.object({
+     *   title: z.string(),
+     *   price: z.number(),
+     *   inStock: z.boolean()
+     * });
+     *
+     * const doc = await client.scrape('https://example.com/product', {
+     *   engine: 'smart',
+     *   extract: {
+     *     schema,
+     *     systemPrompt: 'Extract product details',
+     *     provider: 'openai'
+     *   }
+     * });
+     * console.log(doc.extract?.data);
+     * ```
+     */
+    scrape(url: string, options?: ScrapeOptions): Promise<Document>;
+    /**
+     * Start a batch scrape job (async)
+     *
+     * @param urls - Array of URLs to scrape
+     * @param options - Batch scrape options
+     * @returns Batch job ID and initial status
+     *
+     * @example
+     * ```typescript
+     * const { id } = await client.startBatchScrape(
+     *   ['https://a.com', 'https://b.com', 'https://c.com'],
+     *   { options: { formats: ['markdown'] } }
+     * );
+     *
+     * // Poll manually
+     * let status = await client.getBatchScrapeStatus(id);
+     * while (status.status === 'scraping') {
+     *   await new Promise(r => setTimeout(r, 2000));
+     *   status = await client.getBatchScrapeStatus(id);
+     * }
+     * ```
+     */
+    startBatchScrape(urls: string[], options?: BatchScrapeOptions): Promise<BatchScrapeResponse>;
+    /**
+     * Get batch scrape job status and data
+     *
+     * @param jobId - Batch job ID
+     * @returns Current job status and scraped data
+     */
+    getBatchScrapeStatus(jobId: string): Promise<BatchScrapeJob>;
+    /**
+     * Cancel a batch scrape job
+     *
+     * @param jobId - Batch job ID
+     * @returns True if cancelled successfully
+     */
+    cancelBatchScrape(jobId: string): Promise<boolean>;
+    /**
+     * Get batch scrape job errors
+     *
+     * @param jobId - Batch job ID
+     * @returns Errors and robots.txt blocked URLs
+     */
+    getBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse>;
+    /**
+     * Batch scrape multiple URLs and wait for completion
+     *
+     * @param urls - Array of URLs to scrape
+     * @param options - Batch options including pollInterval and timeout
+     * @returns Final job with all scraped data
+     *
+     * @example
+     * ```typescript
+     * const job = await client.batchScrape(
+     *   ['https://a.com', 'https://b.com', 'https://c.com'],
+     *   {
+     *     options: { formats: ['markdown'], engine: 'smart' },
+     *     pollInterval: 2000,
+     *     timeout: 300
+     *   }
+     * );
+     *
+     * console.log(`Scraped ${job.completed} URLs`);
+     * job.data.forEach(doc => console.log(doc.url, doc.markdown?.length));
+     * ```
+     */
+    batchScrape(urls: string[], options?: BatchScrapeOptions): Promise<BatchScrapeJob>;
+    /**
+     * Start a crawl job (async)
+     *
+     * Use this method when you want to start a crawl and manage polling yourself.
+     * For automatic polling, use the `crawl()` method instead.
+     *
+     * @param url - Root URL to crawl
+     * @param options - Crawl options
+     * @returns Crawl job ID and initial status
+     *
+     * @example
+     * ```typescript
+     * const { id } = await client.startCrawl('https://example.com', {
+     *   limit: 10,
+     *   engine: 'dynamic'
+     * });
+     *
+     * // Poll for status manually
+     * let status = await client.getCrawlStatus(id);
+     * while (status.status === 'scraping') {
+     *   await new Promise(r => setTimeout(r, 2000));
+     *   status = await client.getCrawlStatus(id);
+     * }
+     * ```
+     */
+    startCrawl(url: string, options?: CrawlOptions): Promise<CrawlResponse>;
+    /**
+     * Get crawl job status and data
+     *
+     * @param jobId - Crawl job ID
+     * @returns Current job status and scraped data
+     */
+    getCrawlStatus(jobId: string): Promise<CrawlJob>;
+    /**
+     * Cancel a crawl job
+     *
+     * @param jobId - Crawl job ID
+     * @returns True if cancelled successfully
+     */
+    cancelCrawl(jobId: string): Promise<boolean>;
+    /**
+     * Get crawl job errors and robots.txt blocks
+     *
+     * @param jobId - Crawl job ID
+     * @returns Errors and robots.txt blocked URLs
+     */
+    getCrawlErrors(jobId: string): Promise<CrawlErrorsResponse>;
+    /**
+     * Crawl a website and wait for completion
+     *
+     * This method starts a crawl job and automatically polls until completion.
+     *
+     * @param url - Root URL to crawl
+     * @param options - Crawl options including pollInterval and timeout
+     * @returns Final crawl job with all scraped data
+     *
+     * @example
+     * ```typescript
+     * const job = await client.crawl('https://example.com', {
+     *   limit: 10,
+     *   engine: 'dynamic',
+     *   formats: ['markdown'],
+     *   pollInterval: 2000, // Poll every 2 seconds
+     *   timeout: 300 // 5 minute timeout
+     * });
+     *
+     * console.log(`Crawled ${job.completed} pages`);
+     * job.data.forEach(doc => console.log(doc.url));
+     * ```
+     */
+    crawl(url: string, options?: CrawlOptions): Promise<CrawlJob>;
+    /**
+     * Start an extract job (async)
+     *
+     * @param options - Extract request options
+     * @returns Extract job ID or immediate result
+     *
+     * @example
+     * ```typescript
+     * const { id } = await client.startExtract({
+     *   urls: ['https://example.com/product'],
+     *   schema: { name: 'string', price: 'number' },
+     *   provider: 'openai'
+     * });
+     *
+     * // Poll manually
+     * let status = await client.getExtractStatus(id);
+     * while (status.status === 'processing') {
+     *   await new Promise(r => setTimeout(r, 2000));
+     *   status = await client.getExtractStatus(id);
+     * }
+     * console.log(status.data);
+     * ```
+     */
+    startExtract(options: ExtractRequestOptions): Promise<ExtractResponse>;
+    /**
+     * Get extract job status and data
+     *
+     * @param jobId - Extract job ID
+     * @returns Current job status and extracted data
+     */
+    getExtractStatus(jobId: string): Promise<ExtractResponse>;
+    /**
+     * Extract structured data from URLs using LLM and wait for completion
+     *
+     * @param options - Extract options including schema, prompt, and timeout
+     * @returns Final extract result with structured data
+     *
+     * @example With Zod schema
+     * ```typescript
+     * import { z } from 'zod';
+     *
+     * const result = await client.extract({
+     *   urls: ['https://example.com/product'],
+     *   schema: z.object({
+     *     name: z.string(),
+     *     price: z.number(),
+     *     inStock: z.boolean(),
+     *     features: z.array(z.string())
+     *   }),
+     *   systemPrompt: 'Extract product information from the page',
+     *   provider: 'openai',
+     *   timeout: 60
+     * });
+     *
+     * console.log(result.data);
+     * ```
+     *
+     * @example With natural language prompt
+     * ```typescript
+     * const result = await client.extract({
+     *   urls: ['https://example.com/about'],
+     *   prompt: 'Extract the company name, founding year, and list of team members',
+     *   enableWebSearch: true
+     * });
+     *
+     * console.log(result.data);
+     * ```
+     */
+    extract(options: ExtractRequestOptions): Promise<ExtractResponse>;
+    /**
+     * Map a website to discover all URLs
+     *
+     * @param url - Root URL to map
+     * @param options - Map options
+     * @returns List of discovered URLs
+     *
+     * @example
+     * ```typescript
+     * const result = await client.map('https://example.com', {
+     *   engine: 'dynamic'
+     * });
+     *
+     * console.log(`Found ${result.count} URLs:`);
+     * result.links.forEach(url => console.log(url));
+     * ```
+     */
+    map(url: string, options?: MapOptions): Promise<MapResponse>;
+    /**
+     * Search the web and optionally scrape results
+     *
+     * @param query - Search query
+     * @param options - Search options
+     * @returns Search results with optional scraped content
+     *
+     * @example Basic search
+     * ```typescript
+     * const results = await client.search('best restaurants in NYC', {
+     *   limit: 10,
+     *   lang: 'en',
+     *   country: 'us'
+     * });
+     *
+     * results.data.forEach(r => {
+     *   console.log(`${r.title}: ${r.url}`);
+     * });
+     * ```
+     *
+     * @example Search with scraping
+     * ```typescript
+     * const results = await client.search('best laptops 2024', {
+     *   limit: 5,
+     *   scrapeOptions: {
+     *     formats: ['markdown']
+     *   },
+     *   engine: 'smart'
+     * });
+     *
+     * results.data.forEach(r => {
+     *   console.log(r.title);
+     *   console.log(r.markdown?.substring(0, 200));
+     * });
+     * ```
+     *
+     * @example Search with LLM extraction
+     * ```typescript
+     * import { z } from 'zod';
+     *
+     * const results = await client.search('iPhone 15 Pro reviews', {
+     *   limit: 5,
+     *   scrapeOptions: { formats: ['markdown'] },
+     *   extract: {
+     *     schema: z.object({
+     *       pros: z.array(z.string()),
+     *       cons: z.array(z.string()),
+     *       rating: z.number()
+     *     }),
+     *     systemPrompt: 'Extract review summary from the content'
+     *   }
+     * });
+     *
+     * console.log(results.extract?.data);
+     * ```
+     */
+    search(query: string, options?: SearchOptions): Promise<SearchResponse>;
+    /**
+     * Get current concurrency usage
+     *
+     * @returns Current and max concurrency
+     *
+     * @example
+     * ```typescript
+     * const { concurrency, maxConcurrency } = await client.getConcurrency();
+     * console.log(`Using ${concurrency}/${maxConcurrency} concurrent requests`);
+     * ```
+     */
+    getConcurrency(): Promise<ConcurrencyInfo>;
+    /**
+     * Get current credit usage
+     *
+     * @returns Credit usage information
+     *
+     * @example
+     * ```typescript
+     * const credits = await client.getCreditUsage();
+     * console.log(`Remaining credits: ${credits.remainingCredits}`);
+     * ```
+     */
+    getCreditUsage(): Promise<CreditUsage>;
+    /**
+     * Get current token usage (for LLM extraction)
+     *
+     * @returns Token usage information
+     *
+     * @example
+     * ```typescript
+     * const tokens = await client.getTokenUsage();
+     * console.log(`Remaining tokens: ${tokens.remainingTokens}`);
+     * ```
+     */
+    getTokenUsage(): Promise<TokenUsage>;
+    /**
+     * Get queue status information
+     *
+     * @returns Queue status metrics
+     *
+     * @example
+     * ```typescript
+     * const queue = await client.getQueueStatus();
+     * console.log(`Jobs in queue: ${queue.jobsInQueue}`);
+     * console.log(`Active: ${queue.activeJobsInQueue}, Waiting: ${queue.waitingJobsInQueue}`);
+     * ```
+     */
+    getQueueStatus(): Promise<QueueStatus>;
+}
+
+/**
+ * Base error class for CrawlGate SDK errors
+ */
+declare class CrawlGateError extends Error {
+    /**
+     * HTTP status code (if applicable)
+     */
+    readonly statusCode?: number;
+    /**
+     * Error code for programmatic handling
+     */
+    readonly code?: string;
+    /**
+     * Additional error details
+     */
+    readonly details?: unknown;
+    constructor(message: string, statusCode?: number, code?: string, details?: unknown);
+}
+/**
+ * Error thrown when authentication fails
+ */
+declare class AuthenticationError extends CrawlGateError {
+    constructor(message?: string);
+}
+/**
+ * Error thrown when request validation fails
+ */
+declare class ValidationError extends CrawlGateError {
+    constructor(message: string, details?: unknown);
+}
+/**
+ * Error thrown when a crawl job times out
+ */
+declare class JobTimeoutError extends CrawlGateError {
+    /**
+     * Job ID that timed out
+     */
+    readonly jobId: string;
+    /**
+     * Timeout duration in seconds
+     */
+    readonly timeoutSeconds: number;
+    constructor(jobId: string, timeoutSeconds: number);
+}
+/**
+ * Error thrown when upstream service is unavailable
+ */
+declare class ServiceUnavailableError extends CrawlGateError {
+    constructor(message?: string);
+}
+/**
+ * Error thrown when rate limit is exceeded
+ */
+declare class RateLimitError extends CrawlGateError {
+    /**
+     * Time to wait before retrying (in seconds)
+     */
+    readonly retryAfter?: number;
+    constructor(message?: string, retryAfter?: number);
+}
+/**
+ * Error thrown when LLM extraction fails
+ */
+declare class ExtractionError extends CrawlGateError {
+    /**
+     * Provider that failed
+     */
+    readonly provider?: string;
+    constructor(message: string, provider?: string);
+}
+
+export { AuthenticationError, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse, type ConcurrencyInfo, type CrawlError, type CrawlErrorsResponse, CrawlGateClient, type CrawlGateClientOptions, CrawlGateError, type CrawlJob, type CrawlOptions, type CrawlResponse, type CrawlStatus, type CreditUsage, type Document, type DocumentMetadata, type Engine, type ExtractOptions, type ExtractRequestOptions, type ExtractResponse, type ExtractResult, type ExtractStatus, ExtractionError, type FormatType, JobTimeoutError, type JsonSchema, type LLMProvider, type MapOptions, type MapResponse, type PaginationConfig, type ProxyOption, type QueueStatus, RateLimitError, type ScrapeOptions, type ScrapeResponse, type SearchOptions, type SearchResponse, type SearchResult, ServiceUnavailableError, type TokenUsage, ValidationError, type WebhookConfig, CrawlGateClient as default };