@tothalex/cloud 0.0.44 → 0.0.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tothalex/cloud",
-  "version": "0.0.44",
+  "version": "0.0.46",
   "main": "",
   "types": "./src/index.d.ts",
   "files": [

package/src/cloud/spider.d.ts

@@ -0,0 +1,122 @@
+declare module 'cloud/spider' {
+  /** The rendering driver to use */
+  type SpiderDriver = 'http' | 'chrome' | 'smart'
+
+  /** Redirect policy */
+  type RedirectPolicy = 'loose' | 'strict' | 'none'
+
+  /** Options shared by all spider functions */
+  interface SpiderBaseOptions {
+    /** Rendering driver: 'http' (default), 'chrome' (headless Chrome), or 'smart' (auto-detect) */
+    driver?: SpiderDriver
+    /** Custom HTTP headers */
+    headers?: Record<string, string>
+    /** User-Agent string */
+    userAgent?: string
+    /** Proxy URL (e.g., 'http://user:pass@proxy:8080', supports SOCKS) */
+    proxy?: string
+    /** Cookie string (e.g., 'foo=bar; baz=qux') */
+    cookies?: string
+    /** Per-request timeout in milliseconds (default: 15000) */
+    requestTimeout?: number
+    /** Accept invalid TLS certificates (default: false) */
+    acceptInsecureCerts?: boolean
+    /** Respect robots.txt (default: true for crawl/scrape, false for fetchPage) */
+    respectRobotsTxt?: boolean
+  }
+
+  /** Options for scrape and crawl */
+  interface CrawlOptions extends SpiderBaseOptions {
+    /** Maximum number of pages to visit */
+    limit?: number
+    /** Maximum crawl depth from the start URL (default: 25) */
+    depth?: number
+    /** Polite delay between requests in milliseconds */
+    delay?: number
+    /** Maximum concurrent requests */
+    concurrency?: number
+    /** Overall crawl timeout in milliseconds */
+    crawlTimeout?: number
+    /** Include subdomains (default: false) */
+    subdomains?: boolean
+    /** Include all TLDs for the domain (default: false) */
+    tld?: boolean
+    /** URL patterns to allow (regex; others are excluded) */
+    allowUrls?: string[]
+    /** URL patterns to block (regex) */
+    blockUrls?: string[]
+    /** Redirect policy: 'loose' (default), 'strict', 'none' */
+    redirectPolicy?: RedirectPolicy
+    /** Only crawl HTML pages, skip other resources (default: false) */
+    onlyHtml?: boolean
+
+    /** Wait for a CSS selector to appear before returning (chrome only) */
+    waitForSelector?: string
+    /** Wait for a fixed delay in milliseconds before returning (chrome only) */
+    waitForDelay?: number
+    /** Max timeout in milliseconds for all waitFor operations (chrome only, default: 30000) */
+    waitForTimeout?: number
+    /** Wait for network to become idle before returning (chrome only) */
+    waitForIdleNetwork?: boolean
+    /** Wait for page navigations/redirects to complete before returning (chrome only) */
+    waitForPageNavigations?: boolean
+  }
+
+  /** A crawled page with its content */
+  interface SpiderPage {
+    /** The URL of the page */
+    url: string
+    /** The HTML content */
+    html: string
+    /** HTTP status code */
+    status: number
+    /** Response headers */
+    headers: Record<string, string>
+    /** Final URL after redirects (if different from url) */
+    finalUrl?: string
+    /** Error message if the request failed */
+    error?: string
+  }
+
+  /**
+   * Fetch a single page and return its content.
+   * Defaults to HTTP driver with no robots.txt check.
+   */
+  export function fetchPage(url: string, options?: SpiderBaseOptions): Promise<SpiderPage>
+
+  /**
+   * Crawl a website and return all pages with their HTML content.
+   */
+  export function scrape(url: string, options?: CrawlOptions): Promise<SpiderPage[]>
+
+  /**
+   * Crawl a website and return only the discovered URLs.
+   */
+  export function crawl(url: string, options?: CrawlOptions): Promise<string[]>
+
+  /**
+   * Stream-crawl a website, calling the callback for each page as it arrives.
+   * Returns the total number of pages streamed.
+   *
+   * @example
+   * ```js
+   * const count = await scrapeStream('https://example.com', { limit: 100 }, (page) => {
+   *   console.log(page.url, page.status)
+   * })
+   * ```
+   */
+  export function scrapeStream(
+    url: string,
+    options: CrawlOptions | undefined,
+    callback: (page: SpiderPage) => void,
+  ): Promise<number>
+
+  const spider: {
+    fetchPage: typeof fetchPage
+    scrape: typeof scrape
+    crawl: typeof crawl
+    scrapeStream: typeof scrapeStream
+  }
+
+  export default spider
+}

package/src/index.d.ts

@@ -5,6 +5,7 @@
 /// <reference types="./cloud/postgres.d.ts" />
 /// <reference types="./cloud/secret.d.ts" />
 /// <reference types="./cloud/uuid.d.ts" />
+/// <reference types="./cloud/spider.d.ts" />
 
 /// <reference types="./assert.d.ts" />
 /// <reference types="./buffer.d.ts" />