spidercloud 1.0.0

data/lib/spider_cloud.rb

@@ -0,0 +1,37 @@
+require 'json'
+require 'base64'
+require 'uri'
+
+require 'faraday'
+require 'dynamic_schema'
+
+require_relative 'spider_cloud/version'
+
+require_relative 'spider_cloud/helpers'
+require_relative 'spider_cloud/shared_schemas'
+require_relative 'spider_cloud/costs'
+require_relative 'spider_cloud/error_result'
+require_relative 'spider_cloud/request'
+require_relative 'spider_cloud/response_methods'
+
+require_relative 'spider_cloud/scrape_options'
+require_relative 'spider_cloud/scrape_result'
+require_relative 'spider_cloud/scrape_request'
+
+require_relative 'spider_cloud/crawl_options'
+require_relative 'spider_cloud/crawl_result'
+require_relative 'spider_cloud/crawl_request'
+
+require_relative 'spider_cloud/screenshot_options'
+require_relative 'spider_cloud/screenshot_result'
+require_relative 'spider_cloud/screenshot_request'
+
+require_relative 'spider_cloud/links_options'
+require_relative 'spider_cloud/links_result'
+require_relative 'spider_cloud/links_request'
+
+require_relative 'spider_cloud/module_methods'
+
+module SpiderCloud
+  extend ModuleMethods
+end

data/lib/spidercloud.rb

@@ -0,0 +1 @@
+require_relative 'spider_cloud'

data/readme/crawl.md

@@ -0,0 +1,218 @@
+# Crawl Endpoint
+
+The Crawl endpoint discovers and extracts content from multiple pages starting
+from a given URL.
+
+**API Reference:** https://spider.cloud/docs/api#crawl
+
+## Basic Usage
+
+```ruby
+# Always use a limit to control credit usage
+response = SpiderCloud.crawl( 'https://example.com', limit: 5 )
+
+response.result.each do | page |
+  puts "#{ page.url }: #{ page.content&.length } chars"
+end
+```
+
+## With Options
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 5                    # max pages to crawl
+  depth 2                    # max link depth
+  return_format :markdown
+  readability true
+end
+
+response = SpiderCloud.crawl( 'https://example.com', options )
+```
+
+## Options Reference
+
+### Core Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `limit` | Integer | 0 | Max pages to crawl (0 = unlimited) |
+| `depth` | Integer | 25 | Max crawl depth |
+| `return_format` | Symbol | `:raw` | Output format |
+| `request` | Symbol | `:smart` | Request type |
+
+### Crawl Scope
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `subdomains` | Boolean | Include subdomains |
+| `tld` | Boolean | Include TLD variations |
+| `external_domains` | Array | External domains to include (`["*"]` for all) |
+| `redirect_policy` | Symbol | `:loose`, `:strict`, `:none` |
+
+### URL Filtering
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `blacklist` | Array | Paths to exclude (regex supported) |
+| `whitelist` | Array | Paths to include only |
+| `budget` | Hash | Path-based page limits |
+| `link_rewrite` | Hash | URL rewrite rules |
+
+### Budget Example
+
+Control how many pages to crawl per path:
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 100
+  budget( {
+    '*' => 5,        # default: 5 pages per path
+    '/docs/' => 50,  # up to 50 pages under /docs/
+    '/blog/' => 20   # up to 20 pages under /blog/
+  } )
+end
+```
+
+### Sitemap Options
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `sitemap` | Boolean | Use sitemap for discovery |
+| `sitemap_only` | Boolean | Only crawl sitemap URLs |
+| `sitemap_path` | String | Custom sitemap path |
+
+### Content Extraction
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `readability` | Boolean | Safari Reader Mode |
+| `root_selector` | String | CSS selector for content |
+| `exclude_selector` | String | CSS selector to ignore |
+| `css_extraction_map` | Hash | Structured data extraction |
+| `filter_main_only` | Boolean | Main content only |
+| `full_resources` | Boolean | Download images, videos |
+
+### Output Options
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `return_json_data` | Boolean | Return SSR JSON data |
+| `return_headers` | Boolean | Include HTTP headers |
+| `return_cookies` | Boolean | Include cookies |
+| `return_page_links` | Boolean | Include discovered links |
+| `return_embeddings` | Boolean | Include embeddings |
+| `metadata` | Boolean | Collect page metadata |
+
+### Performance
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `request_timeout` | Integer | Timeout per page (5-255 seconds) |
+| `cache` | Boolean | Enable caching |
+| `concurrency_limit` | Integer | Concurrent requests |
+| `delay` | Integer | Delay between requests (ms) |
+
+### Cost Control
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `max_credits_per_page` | Integer | Max credits per page |
+| `max_credits_allowed` | Integer | Total credit limit |
+| `crawl_timeout` | Hash | Max crawl duration `{seconds:, nanoseconds:}` |
+
+### Webhooks
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 100
+  webhooks do
+    destination 'https://your-server.com/webhook'
+    on_credits_depleted true
+    on_find true
+  end
+end
+```
+
+## Response
+
+```ruby
+response = SpiderCloud.crawl( 'https://example.com', limit: 5 )
+
+response.result.success?    # => true
+response.result.count       # => 5
+response.result.urls        # => ["https://...", ...]
+response.result.contents    # => ["...", ...]
+response.result.total_cost  # => 0.0002
+
+# Iterate over pages
+response.result.each do | page |
+  page.url                  # => "https://..."
+  page.content              # => "..."
+  page.status               # => 200
+  page.costs.total_cost     # => 0.00004
+end
+
+# Filter by success
+response.result.succeeded   # => [successful pages]
+response.result.failed      # => [failed pages]
+```
+
+## Examples
+
+### Crawl Documentation
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 50
+  whitelist [ '/docs/' ]
+  return_format :markdown
+  readability true
+end
+
+response = SpiderCloud.crawl( 'https://example.com', options )
+```
+
+### Crawl with Depth Limit
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 20
+  depth 2
+end
+
+response = SpiderCloud.crawl( 'https://example.com', options )
+```
+
+### Exclude Paths
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 50
+  blacklist [ '/admin/', '/private/', '/api/' ]
+end
+```
+
+### Use Sitemap
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 100
+  sitemap true
+  sitemap_only true
+end
+```
+
+### With Automation
+
+```ruby
+options = SpiderCloud::CrawlOptions.build do
+  limit 10
+  automation_scripts( {
+    '/login' => [
+      { 'Fill' => { 'selector' => '#email', 'value' => 'user@example.com' } },
+      { 'Click' => 'button[type=submit]' },
+      { 'WaitForNavigation' => true }
+    ]
+  } )
+end
+```

data/readme/links.md

@@ -0,0 +1,198 @@
+# Links Endpoint
+
+The Links endpoint discovers and extracts all links from a website without
+returning page content. This is more efficient than crawling when you only
+need URLs.
+
+**API Reference:** https://spider.cloud/docs/api#links
+
+## Basic Usage
+
+```ruby
+# Always use a limit to control credit usage
+response = SpiderCloud.links( 'https://example.com', limit: 5 )
+
+response.result.urls.each do | url |
+  puts url
+end
+```
+
+## With Options
+
+```ruby
+options = SpiderCloud::LinksOptions.build do
+  limit 10
+  depth 2
+  subdomains false
+end
+
+response = SpiderCloud.links( 'https://example.com', options )
+```
+
+## Options Reference
+
+### Core Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `limit` | Integer | 0 | Max pages to check (0 = unlimited) |
+| `depth` | Integer | 25 | Max link depth |
+| `request` | Symbol | `:smart` | Request type |
+| `request_timeout` | Integer | 60 | Timeout per request (5-255 seconds) |
+
+### Scope
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `subdomains` | Boolean | Include subdomains |
+| `tld` | Boolean | Include TLD variations |
+| `external_domains` | Array | External domains to include |
+| `redirect_policy` | Symbol | `:loose`, `:strict`, `:none` |
+
+### URL Filtering
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `blacklist` | Array | Paths to exclude (regex supported) |
+| `whitelist` | Array | Paths to include only |
+| `budget` | Hash | Path-based page limits |
+
+### Sitemap
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `sitemap` | Boolean | Use sitemap for discovery |
+| `sitemap_only` | Boolean | Only check sitemap URLs |
+| `sitemap_path` | String | Custom sitemap path |
+
+### Performance
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `cache` | Boolean | Enable caching |
+| `respect_robots` | Boolean | Respect robots.txt |
+
+### Proxy
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `proxy` | Symbol | Proxy pool: `:residential`, `:mobile`, `:isp` |
+| `proxy_enabled` | Boolean | Enable proxy |
+| `country_code` | String | ISO country code |
+
+## Response
+
+```ruby
+response = SpiderCloud.links( 'https://example.com', limit: 5 )
+
+response.result.success?    # => true
+response.result.count       # => 5
+response.result.urls        # => ["https://...", ...]
+
+# Iterate over results
+response.result.each do | item |
+  item.url                  # => "https://..."
+  item.status               # => 200
+  item.duration_elapsed_ms  # => 123
+  item.error                # => nil
+end
+
+# Filter results
+response.result.succeeded   # => [successful items]
+response.result.failed      # => [failed items]
+response.result.with_status( 200 )  # => [items with 200 status]
+```
+
+## Examples
+
+### Discover All Links
+
+```ruby
+options = SpiderCloud::LinksOptions.build do
+  limit 100
+  depth 3
+end
+
+response = SpiderCloud.links( 'https://example.com', options )
+puts "Found #{ response.result.count } URLs"
+```
+
+### Filter by Path
+
+```ruby
+options = SpiderCloud::LinksOptions.build do
+  limit 50
+  whitelist [ '/docs/', '/api/' ]
+end
+
+response = SpiderCloud.links( 'https://example.com', options )
+```
+
+### Exclude Paths
+
+```ruby
+options = SpiderCloud::LinksOptions.build do
+  limit 50
+  blacklist [ '/admin/', '/private/', '/cdn/' ]
+end
+
+response = SpiderCloud.links( 'https://example.com', options )
+```
+
+### Use Sitemap
+
+```ruby
+options = SpiderCloud::LinksOptions.build do
+  limit 100
+  sitemap true
+  sitemap_only true
+end
+
+response = SpiderCloud.links( 'https://example.com', options )
+```
+
+### Include Subdomains
+
+```ruby
+options = SpiderCloud::LinksOptions.build do
+  limit 50
+  subdomains true
+end
+
+response = SpiderCloud.links( 'https://example.com', options )
+```
+
+### With Budget
+
+```ruby
+options = SpiderCloud::LinksOptions.build do
+  limit 100
+  budget( {
+    '*' => 5,
+    '/docs/' => 50,
+    '/blog/' => 20
+  } )
+end
+
+response = SpiderCloud.links( 'https://example.com', options )
+```
+
+### Get Only Successful URLs
+
+```ruby
+response = SpiderCloud.links( 'https://example.com', limit: 20 )
+
+successful_urls = response.result.succeeded.map( &:url )
+puts successful_urls
+```
+
+### Check for Broken Links
+
+```ruby
+response = SpiderCloud.links( 'https://example.com', limit: 50 )
+
+broken = response.result.failed
+broken.each do | item |
+  puts "Broken: #{ item.url } - Status: #{ item.status }"
+end
+```

data/readme/scrape.md

@@ -0,0 +1,248 @@
+# Scrape Endpoint
+
+The Scrape endpoint extracts content from a single URL with full JavaScript
+rendering support.
+
+**API Reference:** https://spider.cloud/docs/api#scrape
+
+## Basic Usage
+
+```ruby
+response = SpiderCloud.scrape( 'https://example.com' )
+puts response.result.content
+```
+
+## With Options
+
+```ruby
+options = SpiderCloud::ScrapeOptions.build do
+  return_format :markdown
+  readability true
+  metadata true
+end
+
+response = SpiderCloud.scrape( 'https://example.com', options )
+```
+
+## Understanding the Response
+
+The scrape endpoint takes a single URL and returns a `ScrapeResult` containing
+an array of result items. For single URL scrapes, convenience methods delegate
+to the first item:
+
+```ruby
+response = SpiderCloud.scrape( 'https://example.com' )
+
+# the result is enumerable
+response.result.count             # => 1
+
+# convenience methods for single URL scrapes
+response.result.success?          # => true
+response.result.content           # => "# Page Title..."
+response.result.url               # => "https://example.com"
+response.result.status            # => 200
+response.result.error             # => nil
+response.result.costs             # => Costs object
+response.result.duration_elapsed_ms # => 1234
+
+# access the underlying item directly
+response.result.first.content     # same as response.result.content
+```
+
+### Iterating Results
+
+The result is enumerable, so you can iterate over items:
+
+```ruby
+response.result.each do | item |
+  puts item.url
+  puts item.content
+  puts item.status
+end
+```
+
+### Checking Success
+
+```ruby
+# check if the HTTP request succeeded
+response.success?                 # => true/false
+
+# check if the scrape operation succeeded
+response.result.success?          # => true/false
+```
+
+## Options Reference
+
+### Core Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `request` | Symbol | `:smart` | Request type: `:http`, `:chrome`, `:smart` |
+| `return_format` | Symbol | `:raw` | Output format (see formats below) |
+| `request_timeout` | Integer | 60 | Timeout in seconds (5-255) |
+| `lite_mode` | Boolean | false | 50% cost reduction mode |
+
+### Return Formats
+
+- `:markdown` - Markdown format
+- `:commonmark` - CommonMark format
+- `:raw` - Raw HTML
+- `:text` - Plain text
+- `:html2text` - HTML to text
+- `:xml` - XML format
+- `:bytes` - Raw bytes
+- `:empty` - No content
+
+### Content Extraction
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `readability` | Boolean | Safari Reader Mode algorithm |
+| `root_selector` | String | CSS selector for content extraction |
+| `exclude_selector` | String | CSS selector to ignore |
+| `css_extraction_map` | Hash | Structured data extraction |
+| `clean_html` | Boolean | Clean HTML output |
+| `filter_svg` | Boolean | Filter SVG elements |
+| `filter_images` | Boolean | Filter image elements |
+| `filter_main_only` | Boolean | Extract main content only |
+
+### Output Options
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `return_json_data` | Boolean | Return SSR JSON data |
+| `return_headers` | Boolean | Include HTTP headers |
+| `return_cookies` | Boolean | Include cookies |
+| `return_page_links` | Boolean | Include discovered links |
+| `return_embeddings` | Boolean | Include OpenAI embeddings |
+| `metadata` | Boolean | Collect page metadata |
+
+### Network Configuration
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `network_blacklist` | Array | Block matching network requests |
+| `network_whitelist` | Array | Allow only matching requests |
+| `anti_bot` | Boolean | Enable anti-bot measures |
+
+### Session & Authentication
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `session` | Boolean | Persist session |
+| `cookies` | String | HTTP cookies |
+| `headers` | Hash | Custom HTTP headers |
+| `user_agent` | String | Custom user agent |
+
+### Proxy Configuration
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `proxy` | Symbol | Proxy pool: `:residential`, `:mobile`, `:isp` |
+| `proxy_enabled` | Boolean | Enable proxy |
+| `remote_proxy` | String | External proxy URL |
+| `country_code` | String | ISO country code |
+| `locale` | String | Locale (e.g., `"en-US"`) |
+
+### Browser Configuration
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `fingerprint` | Boolean | Use fingerprint detection |
+| `stealth` | Boolean | Stealth mode |
+| `viewport` | Hash | Browser viewport `{width:, height:}` |
+| `device` | Symbol | Device: `:mobile`, `:tablet`, `:desktop` |
+| `scroll` | Integer | Scroll duration (ms) |
+| `block_ads` | Boolean | Block ads |
+| `block_analytics` | Boolean | Block analytics |
+| `block_stylesheets` | Boolean | Block stylesheets |
+| `block_images` | Boolean | Block images |
+
+### Wait Conditions
+
+```ruby
+wait_for do
+  # Wait for CSS selector
+  selector '#content'
+
+  # Wait for network idle
+  idle_network do
+    timeout { seconds 5; nanoseconds 0 }
+  end
+
+  # Wait for delay
+  delay do
+    timeout { seconds 2; nanoseconds 0 }
+  end
+end
+```
+
+### AI/LLM Integration
+
+```ruby
+gpt_config do
+  prompt 'Summarize this page'
+  model 'gpt-4'
+  max_tokens 1000
+  temperature 0.7
+  top_p 0.9
+end
+```
+
+### Chunking
+
+```ruby
+chunking_algorithm do
+  type :by_words  # :no, :by_words, :by_lines, :by_character_length, :by_sentence
+  value 500
+end
+```
+
+## Examples
+
+### Extract Clean Content
+
+```ruby
+options = SpiderCloud::ScrapeOptions.build do
+  return_format :markdown
+  readability true
+  filter_main_only true
+end
+
+response = SpiderCloud.scrape( 'https://example.com/article', options )
+```
+
+### With CSS Extraction
+
+```ruby
+options = SpiderCloud::ScrapeOptions.build do
+  css_extraction_map( {
+    'title' => 'h1',
+    'price' => '.price',
+    'description' => '.product-description'
+  } )
+end
+```
+
+### With Proxy and Stealth
+
+```ruby
+options = SpiderCloud::ScrapeOptions.build do
+  proxy :residential
+  proxy_enabled true
+  country_code 'US'
+  stealth true
+  fingerprint true
+end
+```
+
+### Wait for Dynamic Content
+
+```ruby
+options = SpiderCloud::ScrapeOptions.build do
+  request :chrome
+  wait_for do
+    selector '.loaded-content'
+  end
+end
+```