@crawlkit-sh/sdk 1.0.0

package/dist/index.cjs

@@ -0,0 +1,745 @@
+'use strict';
+
+// src/errors/index.ts
+var CrawlKitError = class extends Error {
+  constructor(code, message, statusCode, creditsRefunded, creditsRemaining) {
+    super(message);
+    this.name = "CrawlKitError";
+    this.code = code;
+    this.statusCode = statusCode;
+    this.creditsRefunded = creditsRefunded;
+    this.creditsRemaining = creditsRemaining;
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, this.constructor);
+    }
+  }
+};
+var AuthenticationError = class extends CrawlKitError {
+  constructor(message = "Invalid or missing API key") {
+    super("VALIDATION_ERROR", message, 401);
+    this.name = "AuthenticationError";
+  }
+};
+var InsufficientCreditsError = class extends CrawlKitError {
+  constructor(message, creditsRefunded, creditsRemaining) {
+    super("INSUFFICIENT_CREDITS", message, 402, creditsRefunded, creditsRemaining);
+    this.name = "InsufficientCreditsError";
+    this.available = creditsRemaining;
+  }
+};
+var ValidationError = class extends CrawlKitError {
+  constructor(message) {
+    super("VALIDATION_ERROR", message, 400);
+    this.name = "ValidationError";
+  }
+};
+var RateLimitError = class extends CrawlKitError {
+  constructor(message = "Rate limit exceeded") {
+    super("RATE_LIMITED", message, 429);
+    this.name = "RateLimitError";
+  }
+};
+var TimeoutError = class extends CrawlKitError {
+  constructor(message, creditsRefunded, creditsRemaining) {
+    super("TIMEOUT", message, 408, creditsRefunded, creditsRemaining);
+    this.name = "TimeoutError";
+  }
+};
+var NotFoundError = class extends CrawlKitError {
+  constructor(message = "Resource not found") {
+    super("NOT_FOUND", message, 404);
+    this.name = "NotFoundError";
+  }
+};
+var NetworkError = class extends CrawlKitError {
+  constructor(code, message, creditsRefunded, creditsRemaining) {
+    super(code, message, 502, creditsRefunded, creditsRemaining);
+    this.name = "NetworkError";
+  }
+};
+function createErrorFromResponse(code, message, statusCode, creditsRefunded, creditsRemaining) {
+  const errorCode = code;
+  switch (statusCode) {
+    case 401:
+      return new AuthenticationError(message);
+    case 402:
+      return new InsufficientCreditsError(message, creditsRefunded, creditsRemaining);
+    case 429:
+      return new RateLimitError(message);
+    case 404:
+      return new NotFoundError(message);
+  }
+  switch (errorCode) {
+    case "VALIDATION_ERROR":
+      return new ValidationError(message);
+    case "INSUFFICIENT_CREDITS":
+      return new InsufficientCreditsError(message, creditsRefunded, creditsRemaining);
+    case "TIMEOUT":
+      return new TimeoutError(message, creditsRefunded, creditsRemaining);
+    case "RATE_LIMITED":
+      return new RateLimitError(message);
+    case "NOT_FOUND":
+      return new NotFoundError(message);
+    case "DNS_FAILED":
+    case "CONNECTION_REFUSED":
+    case "SSL_ERROR":
+    case "TOO_MANY_REDIRECTS":
+    case "PROXY_ERROR":
+      return new NetworkError(errorCode, message, creditsRefunded, creditsRemaining);
+    default:
+      return new CrawlKitError(
+        errorCode,
+        message,
+        statusCode,
+        creditsRefunded,
+        creditsRemaining
+      );
+  }
+}
+
+// src/resources/base.ts
+var BaseResource = class {
+  constructor(config) {
+    this.config = config;
+  }
+  /**
+   * Make a POST request to the API
+   * @param endpoint - API endpoint path (e.g., '/v1/crawl/scrape')
+   * @param body - Request body object
+   * @returns Parsed response data
+   * @throws {CrawlKitError} On API errors
+   */
+  async post(endpoint, body) {
+    const url = `${this.config.baseUrl}${endpoint}`;
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
+    try {
+      const response = await this.config.fetch(url, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `ApiKey ${this.config.apiKey}`,
+          "User-Agent": "@crawlkit/sdk"
+        },
+        body: JSON.stringify(body),
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      let json;
+      try {
+        json = await response.json();
+      } catch {
+        throw new CrawlKitError(
+          "PARSE_ERROR",
+          "Failed to parse API response",
+          response.status
+        );
+      }
+      if (!json.success) {
+        const errorResponse = json;
+        throw createErrorFromResponse(
+          errorResponse.error.code,
+          errorResponse.error.message,
+          response.status,
+          errorResponse.creditsRefunded,
+          errorResponse.creditsRemaining
+        );
+      }
+      return json.data;
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof CrawlKitError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "AbortError") {
+          throw new CrawlKitError(
+            "TIMEOUT",
+            `Request timed out after ${this.config.timeout}ms`,
+            408
+          );
+        }
+        throw new CrawlKitError(
+          "UNKNOWN",
+          error.message || "An unknown error occurred",
+          500
+        );
+      }
+      throw new CrawlKitError("UNKNOWN", "An unknown error occurred", 500);
+    }
+  }
+  /**
+   * Make a GET request to the API
+   * @param endpoint - API endpoint path
+   * @param params - Query parameters
+   * @returns Parsed response data
+   * @throws {CrawlKitError} On API errors
+   */
+  async get(endpoint, params) {
+    let url = `${this.config.baseUrl}${endpoint}`;
+    if (params) {
+      const searchParams = new URLSearchParams();
+      for (const [key, value] of Object.entries(params)) {
+        if (value !== void 0) {
+          searchParams.append(key, String(value));
+        }
+      }
+      const queryString = searchParams.toString();
+      if (queryString) {
+        url += `?${queryString}`;
+      }
+    }
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
+    try {
+      const response = await this.config.fetch(url, {
+        method: "GET",
+        headers: {
+          "Authorization": `ApiKey ${this.config.apiKey}`,
+          "User-Agent": "@crawlkit/sdk"
+        },
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      let json;
+      try {
+        json = await response.json();
+      } catch {
+        throw new CrawlKitError(
+          "PARSE_ERROR",
+          "Failed to parse API response",
+          response.status
+        );
+      }
+      if (!json.success) {
+        const errorResponse = json;
+        throw createErrorFromResponse(
+          errorResponse.error.code,
+          errorResponse.error.message,
+          response.status,
+          errorResponse.creditsRefunded,
+          errorResponse.creditsRemaining
+        );
+      }
+      return json.data;
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof CrawlKitError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "AbortError") {
+          throw new CrawlKitError(
+            "TIMEOUT",
+            `Request timed out after ${this.config.timeout}ms`,
+            408
+          );
+        }
+        throw new CrawlKitError(
+          "UNKNOWN",
+          error.message || "An unknown error occurred",
+          500
+        );
+      }
+      throw new CrawlKitError("UNKNOWN", "An unknown error occurred", 500);
+    }
+  }
+};
+
+// src/resources/crawl.ts
+var CrawlResource = class extends BaseResource {
+  /**
+   * Scrape a URL and return markdown, HTML, metadata, and links
+   *
+   * @param params - Scrape parameters
+   * @returns Scraped page data including markdown, HTML, metadata, and links
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.scrape({
+   *   url: 'https://example.com',
+   *   options: {
+   *     onlyMainContent: true,
+   *     waitFor: '#content'
+   *   }
+   * });
+   * console.log(result.markdown);
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async scrape(params) {
+    return this.post("/v1/crawl/scrape", params);
+  }
+  /**
+   * Extract structured data from a URL using AI
+   *
+   * @param params - Extract parameters including JSON schema
+   * @returns Extracted structured data along with page content
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * interface Product {
+   *   name: string;
+   *   price: number;
+   * }
+   *
+   * const result = await crawlkit.extract<Product>({
+   *   url: 'https://example.com/product',
+   *   schema: {
+   *     type: 'object',
+   *     properties: {
+   *       name: { type: 'string' },
+   *       price: { type: 'number' }
+   *     }
+   *   }
+   * });
+   * console.log(result.json.name, result.json.price);
+   * ```
+   *
+   * @costs 5 credits
+   */
+  async extract(params) {
+    return this.post("/v1/crawl/extract", params);
+  }
+  /**
+   * Perform a web search using DuckDuckGo
+   *
+   * @param params - Search parameters
+   * @returns Search results with titles, URLs, and snippets
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.search({
+   *   query: 'typescript best practices',
+   *   options: {
+   *     maxResults: 10,
+   *     timeRange: 'w' // Past week
+   *   }
+   * });
+   * result.results.forEach(r => console.log(r.title, r.url));
+   * ```
+   *
+   * @costs 1 credit per page (~10 results)
+   */
+  async search(params) {
+    return this.post("/v1/crawl/search", params);
+  }
+  /**
+   * Take a full-page screenshot of a URL
+   *
+   * @param params - Screenshot parameters
+   * @returns Public URL of the screenshot
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.screenshot({
+   *   url: 'https://example.com',
+   *   options: {
+   *     width: 1920,
+   *     height: 1080,
+   *     waitForSelector: '#content'
+   *   }
+   * });
+   * console.log('Screenshot URL:', result.url);
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async screenshot(params) {
+    return this.post("/v1/crawl/screenshot", params);
+  }
+};
+
+// src/resources/linkedin.ts
+var LinkedInResource = class extends BaseResource {
+  /**
+   * Scrape a LinkedIn company profile
+   *
+   * @param params - Company profile parameters
+   * @returns Company profile data including description, employees, jobs, posts
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.linkedin.company({
+   *   url: 'https://www.linkedin.com/company/openai',
+   *   options: { includeJobs: true }
+   * });
+   * console.log(result.company.name);
+   * console.log(result.company.followers);
+   * console.log(result.company.jobs);
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async company(params) {
+    return this.post("/v1/crawl/linkedin/company", params);
+  }
+  /**
+   * Scrape LinkedIn person profile(s)
+   *
+   * @param params - Person profile parameters (single URL or array of URLs, max 10)
+   * @returns Person profile data for each URL
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * // Single profile
+   * const result = await crawlkit.linkedin.person({
+   *   url: 'https://www.linkedin.com/in/username'
+   * });
+   *
+   * // Multiple profiles (batch)
+   * const batchResult = await crawlkit.linkedin.person({
+   *   url: [
+   *     'https://www.linkedin.com/in/user1',
+   *     'https://www.linkedin.com/in/user2'
+   *   ]
+   * });
+   * console.log(`Success: ${batchResult.successCount}, Failed: ${batchResult.failedCount}`);
+   * ```
+   *
+   * @costs 3 credits per URL
+   */
+  async person(params) {
+    return this.post("/v1/crawl/linkedin/person", params);
+  }
+};
+
+// src/resources/instagram.ts
+var InstagramResource = class extends BaseResource {
+  /**
+   * Scrape an Instagram profile
+   *
+   * @param params - Profile parameters (username or URL)
+   * @returns Profile data including bio, follower count, and recent posts
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.instagram.profile({
+   *   username: 'instagram'
+   * });
+   * console.log(result.profile.full_name);
+   * console.log(result.profile.follower_count);
+   * console.log(result.profile.posts.length);
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async profile(params) {
+    return this.post("/v1/crawl/instagram/profile", params);
+  }
+  /**
+   * Scrape Instagram content (post, reel, or video)
+   *
+   * @param params - Content parameters (shortcode or full URL)
+   * @returns Content data including media URLs, likes, comments, and owner info
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * // Using shortcode
+   * const result = await crawlkit.instagram.content({
+   *   shortcode: 'CxIIgCCq8mg'
+   * });
+   *
+   * // Using full URL
+   * const result = await crawlkit.instagram.content({
+   *   shortcode: 'https://www.instagram.com/p/CxIIgCCq8mg/'
+   * });
+   *
+   * console.log(result.post.like_count);
+   * console.log(result.post.video_url);
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async content(params) {
+    return this.post("/v1/crawl/instagram/content", params);
+  }
+};
+
+// src/resources/appstore.ts
+var AppStoreResource = class extends BaseResource {
+  /**
+   * Fetch Google Play Store reviews for an app
+   *
+   * @param params - Reviews parameters including app ID and optional pagination cursor
+   * @returns Reviews with pagination information
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * // First page
+   * const result = await crawlkit.appstore.playstoreReviews({
+   *   appId: 'com.example.app',
+   *   options: { lang: 'en' }
+   * });
+   *
+   * // Next page
+   * if (result.pagination.hasMore) {
+   *   const nextPage = await crawlkit.appstore.playstoreReviews({
+   *     appId: 'com.example.app',
+   *     cursor: result.pagination.nextCursor
+   *   });
+   * }
+   * ```
+   *
+   * @costs 1 credit per page
+   */
+  async playstoreReviews(params) {
+    return this.post("/v1/crawl/playstore/reviews", params);
+  }
+  /**
+   * Fetch Google Play Store app details
+   *
+   * @param params - App detail parameters
+   * @returns Comprehensive app information including ratings, screenshots, permissions
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.appstore.playstoreDetail({
+   *   appId: 'com.example.app',
+   *   options: { lang: 'en' }
+   * });
+   * console.log(result.appName);
+   * console.log(result.rating);
+   * console.log(result.installs);
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async playstoreDetail(params) {
+    return this.post("/v1/crawl/playstore/detail", params);
+  }
+  /**
+   * Fetch Apple App Store reviews for an app
+   *
+   * @param params - Reviews parameters including app ID and optional pagination cursor
+   * @returns Reviews with pagination information
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * // First page
+   * const result = await crawlkit.appstore.appstoreReviews({
+   *   appId: '123456789',
+   *   options: { lang: 'en' }
+   * });
+   *
+   * // Paginate through all reviews
+   * let cursor = result.pagination.nextCursor;
+   * while (cursor) {
+   *   const nextPage = await crawlkit.appstore.appstoreReviews({
+   *     appId: '123456789',
+   *     cursor
+   *   });
+   *   cursor = nextPage.pagination.nextCursor;
+   * }
+   * ```
+   *
+   * @costs 1 credit per page
+   */
+  async appstoreReviews(params) {
+    return this.post("/v1/crawl/appstore/reviews", params);
+  }
+};
+
+// src/client.ts
+var CrawlKit = class {
+  /**
+   * Create a new CrawlKit client
+   *
+   * @param config - Client configuration
+   * @throws {AuthenticationError} If API key is invalid or missing
+   *
+   * @example
+   * ```typescript
+   * const crawlkit = new CrawlKit({
+   *   apiKey: 'ck_your_api_key',
+   *   timeout: 60000 // 60 seconds
+   * });
+   * ```
+   */
+  constructor(config) {
+    if (!config.apiKey) {
+      throw new AuthenticationError("API key is required");
+    }
+    if (!config.apiKey.startsWith("ck_")) {
+      throw new AuthenticationError(
+        'Invalid API key format. API keys must start with "ck_"'
+      );
+    }
+    this.config = {
+      apiKey: config.apiKey,
+      baseUrl: config.baseUrl ?? "https://api.crawlkit.sh",
+      timeout: config.timeout ?? 3e4,
+      fetch: config.fetch ?? globalThis.fetch.bind(globalThis)
+    };
+    const resourceConfig = {
+      apiKey: this.config.apiKey,
+      baseUrl: this.config.baseUrl,
+      timeout: this.config.timeout,
+      fetch: this.config.fetch
+    };
+    this.crawl = new CrawlResource(resourceConfig);
+    this.linkedin = new LinkedInResource(resourceConfig);
+    this.instagram = new InstagramResource(resourceConfig);
+    this.appstore = new AppStoreResource(resourceConfig);
+  }
+  /**
+   * Scrape a URL and return markdown, HTML, metadata, and links
+   *
+   * @param params - Scrape parameters
+   * @returns Scraped page data including markdown, HTML, metadata, and links
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * // Basic scraping
+   * const result = await crawlkit.scrape({
+   *   url: 'https://example.com'
+   * });
+   * console.log(result.markdown);
+   * console.log(result.metadata.title);
+   *
+   * // With browser automation
+   * const spaResult = await crawlkit.scrape({
+   *   url: 'https://example.com/spa',
+   *   options: {
+   *     waitFor: '#content-loaded',
+   *     actions: [
+   *       { type: 'click', selector: '#load-more' },
+   *       { type: 'wait', milliseconds: 2000 }
+   *     ]
+   *   }
+   * });
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async scrape(params) {
+    return this.crawl.scrape(params);
+  }
+  /**
+   * Extract structured data from a URL using AI
+   *
+   * Uses LLM to extract data according to the provided JSON schema.
+   *
+   * @param params - Extract parameters including JSON schema
+   * @returns Extracted structured data along with page content
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * interface Product {
+   *   name: string;
+   *   price: number;
+   *   description: string;
+   *   inStock: boolean;
+   * }
+   *
+   * const result = await crawlkit.extract<Product>({
+   *   url: 'https://example.com/product/123',
+   *   schema: {
+   *     type: 'object',
+   *     properties: {
+   *       name: { type: 'string' },
+   *       price: { type: 'number' },
+   *       description: { type: 'string' },
+   *       inStock: { type: 'boolean' }
+   *     }
+   *   },
+   *   options: {
+   *     prompt: 'Extract product information from this page'
+   *   }
+   * });
+   *
+   * // TypeScript knows result.json is Product
+   * console.log(result.json.name);
+   * console.log(result.json.price);
+   * ```
+   *
+   * @costs 5 credits
+   */
+  async extract(params) {
+    return this.crawl.extract(params);
+  }
+  /**
+   * Perform a web search using DuckDuckGo
+   *
+   * @param params - Search parameters
+   * @returns Search results with titles, URLs, and snippets
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.search({
+   *   query: 'typescript best practices 2024',
+   *   options: {
+   *     maxResults: 20,
+   *     timeRange: 'm', // Past month
+   *     region: 'us-en'
+   *   }
+   * });
+   *
+   * for (const item of result.results) {
+   *   console.log(`${item.position}. ${item.title}`);
+   *   console.log(`   ${item.url}`);
+   *   console.log(`   ${item.snippet}\n`);
+   * }
+   * ```
+   *
+   * @costs 1 credit per page (~10 results)
+   */
+  async search(params) {
+    return this.crawl.search(params);
+  }
+  /**
+   * Take a full-page screenshot of a URL
+   *
+   * @param params - Screenshot parameters
+   * @returns Public URL of the screenshot
+   * @throws {CrawlKitError} On API errors
+   *
+   * @example
+   * ```typescript
+   * const result = await crawlkit.screenshot({
+   *   url: 'https://example.com',
+   *   options: {
+   *     width: 1920,
+   *     height: 1080,
+   *     waitForSelector: '#main-content'
+   *   }
+   * });
+   *
+   * console.log('Screenshot URL:', result.url);
+   * console.log(`Dimensions: ${result.width}x${result.height}`);
+   * ```
+   *
+   * @costs 1 credit
+   */
+  async screenshot(params) {
+    return this.crawl.screenshot(params);
+  }
+};
+
+exports.AuthenticationError = AuthenticationError;
+exports.CrawlKit = CrawlKit;
+exports.CrawlKitError = CrawlKitError;
+exports.InsufficientCreditsError = InsufficientCreditsError;
+exports.NetworkError = NetworkError;
+exports.NotFoundError = NotFoundError;
+exports.RateLimitError = RateLimitError;
+exports.TimeoutError = TimeoutError;
+exports.ValidationError = ValidationError;
+exports.createErrorFromResponse = createErrorFromResponse;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map