zenrows 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af4d111e269c080e71f047c74acec3a319f05e0b6c800cb7067b4d05a23cc209
-  data.tar.gz: 373c8b95c4f311141767d750a509e88ef74dc2e18c947a703b9868c5cce97ad4
+  metadata.gz: 5c04285fb07c401d81848da8a4ba763b3e0cd0a550c5e34fc1716101f229fbb7
+  data.tar.gz: 04d35bac2bafd413d8969d5d61b54f414c54390db0deb014dd721e2874944cc2
 SHA512:
-  metadata.gz: 81b92b61f13e82a6b2462e790df7f009f88d75f7d9cac822dfd5e685fab0c85b85898f50f3ce67b241622609ff02ebf2b98896e0b9a2ed870f41d1ea2231138c
-  data.tar.gz: 9740ef08c21c4d3701795786886df16183d188a0cf6d5fc7d04bc0a2a46b8f00693ec98fecfd4925bf9a2cd69ab9dca93add9e4c54f2631c630f2698444f912a
+  metadata.gz: 12becd343181666b4dc8b8edccc5fffcb410ecaf7ba74b9b7ed8a6859693d20f7b466e27f5e8a245b9841c56d7f2c4b1d2b3fa7a84b241b3725140acc546ad2c
+  data.tar.gz: '0368ed029138f05de5e2041fb4136caacc04d3ae9b172e7b5983f1204eefa0d12afc2ab70724f124922358d201195aa070e60db2248cee7b9031bb9d64e0c6d2'

data/.mcp.json

@@ -0,0 +1,10 @@
+{
+  "mcpServers": {
+    "Context7": {
+      "type": "stdio",
+      "command": "npx",
+      "args": ["-y", "@upstash/context7-mcp@latest"],
+      "env": {}
+    }
+  }
+}

data/.tool-versions

@@ -0,0 +1 @@
+ruby 3.4.2

data/CHANGELOG.md

@@ -7,6 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.1] - 2025-12-25
+
+### Added
+
+- Configurable `api_endpoint` for ApiClient (global config or per-instance)
+- `net_http` backend as fallback when http.rb unavailable
+- GitHub Pages for YARD documentation
+
+### Changed
+
+- SSL context now auto-configured in proxy client (no need to pass `ssl_context:` on every request)
+
+## [0.2.0] - 2025-12-25
+
+### Added
+
+- `ApiClient` for REST API mode (autoparse, css_extractor, markdown output)
+- `CssExtractor` DSL for building extraction rules
+- `ApiResponse` wrapper with typed accessors
+- Proxy options: `device`, `antibot`, `session_ttl`
+- RBS type signatures
+- rubocop-minitest linting
+- Dependabot configuration
+
+### Changed
+
+- Require Ruby >= 3.2.0
+
 ## [0.1.0] - 2025-12-25
 
 ### Added

data/CLAUDE.md

@@ -46,7 +46,7 @@ end
 
 client = Zenrows::Client.new
 http = client.http(js_render: true, premium_proxy: true)
-response = http.get('https://example.com', ssl_context: client.ssl_context)
+response = http.get('https://example.com')  # SSL verification disabled for proxy
 ```
 
 ## Key Options

data/Makefile

@@ -0,0 +1,19 @@
+.PHONY: install test lint rbs docs build
+
+install:
+	bundle install
+
+test:
+	bundle exec rake test
+
+lint:
+	bundle exec rubocop
+
+rbs:
+	rbs -I sig -r monitor -r openssl -r logger validate
+
+docs:
+	bundle exec yard doc
+
+build:
+	bundle exec rake build

data/README.md

@@ -35,12 +35,14 @@ end
 ```ruby
 client = Zenrows::Client.new
 http = client.http(js_render: true, premium_proxy: true)
-response = http.get('https://example.com', ssl_context: client.ssl_context)
+response = http.get('https://example.com')
 
 puts response.body
 puts response.status
 ```
 
+> **Note:** SSL verification is disabled automatically for proxy connections (required by ZenRows).
+
 ### With Options
 
 ```ruby
@@ -70,10 +72,11 @@ instructions = Zenrows::JsInstructions.build do
 end
 
 http = client.http(js_render: true, js_instructions: instructions)
-response = http.get(url, ssl_context: client.ssl_context)
+response = http.get(url)
 ```
 
 Available instructions:
+
 - `click(selector)` - Click element
 - `wait(ms)` - Wait duration
 - `wait_for(selector)` - Wait for element
@@ -108,25 +111,102 @@ http = client.http(
 )
 ```
 
+### Device & Antibot
+
+```ruby
+http = client.http(
+  js_render: true,
+  device: 'mobile',      # mobile/desktop emulation
+  antibot: true          # enhanced antibot bypass
+)
+```
+
+## API Client (v0.2.0+)
+
+For advanced extraction features, use the REST API client:
+
+### Autoparse
+
+Extract structured data from known sites (Amazon, etc.):
+
+```ruby
+api = Zenrows::ApiClient.new
+response = api.get('https://amazon.com/dp/B01LD5GO7I', autoparse: true)
+
+response.parsed  # => { "title" => "...", "price" => "$29.99", ... }
+```
+
+### CSS Extraction
+
+Extract data using CSS selectors:
+
+```ruby
+# Hash syntax
+response = api.get(url, css_extractor: {
+  title: 'h1',
+  links: 'a @href',
+  prices: '.price'
+})
+response.extracted  # => { "title" => "...", "links" => [...], "prices" => [...] }
+
+# DSL syntax
+extractor = Zenrows::CssExtractor.build do
+  extract :title, 'h1'
+  links :urls, 'a.product'
+  images :photos, 'img.gallery'
+end
+response = api.get(url, css_extractor: extractor)
+```
+
+### Markdown Output
+
+```ruby
+response = api.get(url, response_type: 'markdown')
+response.markdown  # => "# Page Title\n\nContent..."
+```
+
+### Response Metadata
+
+```ruby
+response = api.get(url)
+response.status              # => 200
+response.success?            # => true
+response.final_url           # => "https://example.com/redirected"
+response.request_cost        # => 0.001
+response.concurrency_remaining  # => 199
+```
+
 ## Options Reference
 
-| Option | Type | Description |
-|--------|------|-------------|
-| `js_render` | Boolean | Enable JavaScript rendering |
-| `premium_proxy` | Boolean | Use residential proxies |
-| `proxy_country` | String | Country code (us, gb, de, etc.) |
-| `wait` | Integer/Boolean | Wait time in ms (true = 15000) |
-| `wait_for` | String | CSS selector to wait for |
-| `session_id` | Boolean/String | Session persistence |
-| `window_height` | Integer | Browser window height |
-| `window_width` | Integer | Browser window width |
-| `js_instructions` | Array/String | Browser automation |
-| `json_response` | Boolean | Return JSON instead of HTML |
-| `screenshot` | Boolean | Take screenshot |
-| `screenshot_fullpage` | Boolean | Full page screenshot |
-| `screenshot_selector` | String | Screenshot specific element |
-| `block_resources` | String | Block resources (image,media,font) |
-| `headers` | Hash | Custom HTTP headers |
+| Option                | Type            | Description                        |
+| --------------------- | --------------- | ---------------------------------- |
+| `js_render`           | Boolean         | Enable JavaScript rendering        |
+| `premium_proxy`       | Boolean         | Use residential proxies            |
+| `proxy_country`       | String          | Country code (us, gb, de, etc.)    |
+| `device`              | String          | Device emulation (mobile/desktop)  |
+| `antibot`             | Boolean         | Enhanced antibot bypass            |
+| `wait`                | Integer/Boolean | Wait time in ms (true = 15000)     |
+| `wait_for`            | String          | CSS selector to wait for           |
+| `session_id`          | Boolean/String  | Session persistence                |
+| `session_ttl`         | String          | Session duration (1m, 10m, 30m)    |
+| `window_height`       | Integer         | Browser window height              |
+| `window_width`        | Integer         | Browser window width               |
+| `js_instructions`     | Array/String    | Browser automation                 |
+| `json_response`       | Boolean         | Return JSON instead of HTML        |
+| `screenshot`          | Boolean         | Take screenshot                    |
+| `screenshot_fullpage` | Boolean         | Full page screenshot               |
+| `screenshot_selector` | String          | Screenshot specific element        |
+| `block_resources`     | String          | Block resources (image,media,font) |
+| `headers`             | Hash            | Custom HTTP headers                |
+
+### API Client Options
+
+| Option           | Type         | Description                          |
+| ---------------- | ------------ | ------------------------------------ |
+| `autoparse`      | Boolean      | Auto-extract structured data         |
+| `css_extractor`  | Hash/Object  | CSS selectors for extraction         |
+| `response_type`  | String       | Output format ('markdown')           |
+| `outputs`        | String       | Extract specific data (headings,links) |
 
 ## Error Handling
 
@@ -152,6 +232,7 @@ end
 ## Rails Integration
 
 The gem automatically integrates with Rails when detected:
+
 - Uses Rails.logger by default
 - Supports ActiveSupport::Duration for wait times
 

data/lib/zenrows/api_client.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "http"
+require "json"
+require "cgi"
+
+module Zenrows
+  # REST API client for ZenRows Universal Scraper API
+  #
+  # Unlike the proxy-based Client, ApiClient calls the ZenRows API directly.
+  # This enables features not available in proxy mode: autoparse, css_extractor,
+  # response_type (markdown), and outputs.
+  #
+  # @example Basic usage
+  #   api = Zenrows::ApiClient.new
+  #   response = api.get('https://example.com')
+  #
+  # @example With autoparse
+  #   response = api.get('https://amazon.com/dp/B01LD5GO7I', autoparse: true)
+  #   puts response.data  # Structured product data
+  #
+  # @example With CSS extraction
+  #   response = api.get(url, css_extractor: { title: 'h1', links: 'a @href' })
+  #
+  # @example With markdown output
+  #   response = api.get(url, response_type: 'markdown')
+  #
+  # @author Ernest Bursa
+  # @since 0.2.0
+  # @api public
+  class ApiClient
+    # @return [String] ZenRows API key
+    attr_reader :api_key
+
+    # @return [String] API endpoint URL
+    attr_reader :api_endpoint
+
+    # @return [Configuration] Configuration instance
+    attr_reader :config
+
+    # Initialize API client
+    #
+    # @param api_key [String, nil] Override API key (uses global config if nil)
+    # @param api_endpoint [String, nil] Override API endpoint (uses global config if nil)
+    def initialize(api_key: nil, api_endpoint: nil)
+      @config = Zenrows.configuration
+      @api_key = api_key || @config.api_key
+      @api_endpoint = api_endpoint || @config.api_endpoint
+      @config.validate! unless api_key
+    end
+
+    # Make GET request through ZenRows API
+    #
+    # @param url [String] Target URL to scrape
+    # @param options [Hash] Request options
+    # @option options [Boolean] :autoparse Auto-extract structured data
+    # @option options [Hash, CssExtractor] :css_extractor CSS selectors for extraction
+    # @option options [String] :response_type Response format ('markdown')
+    # @option options [String] :outputs Extract specific data ('headings,links,menus')
+    # @option options [Boolean] :js_render Enable JavaScript rendering
+    # @option options [Boolean] :premium_proxy Use residential proxies
+    # @option options [String] :proxy_country Country code
+    # @option options [Integer, Boolean] :wait Wait time in ms
+    # @option options [String] :wait_for CSS selector to wait for
+    # @option options [Boolean, String] :session_id Session persistence
+    # @option options [Array, String] :js_instructions Browser automation
+    # @option options [Boolean] :json_response Return JSON with XHR data
+    # @option options [Boolean] :screenshot Take screenshot
+    # @option options [Boolean] :screenshot_fullpage Full page screenshot
+    # @option options [String] :screenshot_selector Screenshot element
+    # @option options [String] :block_resources Block resources
+    # @option options [String] :device Device emulation
+    # @option options [Boolean] :antibot Enhanced antibot
+    # @return [ApiResponse] Response wrapper
+    # @raise [ConfigurationError] if API key not configured
+    # @raise [AuthenticationError] if API key invalid
+    # @raise [RateLimitError] if rate limited
+    def get(url, **options)
+      params = build_params(url, options)
+      http_response = build_http_client.get(api_endpoint, params: params)
+      handle_response(http_response, options)
+    end
+
+    # Make POST request through ZenRows API
+    #
+    # @param url [String] Target URL
+    # @param body [String, Hash] Request body
+    # @param options [Hash] Request options (same as #get)
+    # @return [ApiResponse] Response wrapper
+    def post(url, body: nil, **options)
+      params = build_params(url, options)
+      http_response = build_http_client.post(api_endpoint, params: params, body: body)
+      handle_response(http_response, options)
+    end
+
+    private
+
+    def build_http_client
+      HTTP
+        .timeout(connect: config.connect_timeout, read: config.read_timeout)
+    end
+
+    def build_params(url, options)
+      params = {apikey: api_key, url: url}
+
+      # API-mode only features
+      params[:autoparse] = "true" if options[:autoparse]
+      params[:response_type] = options[:response_type] if options[:response_type]
+      params[:outputs] = options[:outputs] if options[:outputs]
+
+      if options[:css_extractor]
+        extractor = options[:css_extractor]
+        params[:css_extractor] = extractor.to_json
+      end
+
+      # Common options (also available in proxy mode)
+      params[:js_render] = "true" if options[:js_render]
+      params[:premium_proxy] = "true" if options[:premium_proxy]
+      params[:proxy_country] = options[:proxy_country] if options[:proxy_country]
+      params[:json_response] = "true" if options[:json_response]
+      params[:original_status] = "true" if options[:original_status]
+
+      # Wait options
+      if options[:wait]
+        params[:wait] = (options[:wait] == true) ? 15000 : options[:wait]
+      end
+      params[:wait_for] = options[:wait_for] if options[:wait_for]
+
+      # Session
+      if options[:session_id]
+        params[:session_id] = (options[:session_id] == true) ? rand(1..99999) : options[:session_id]
+      end
+
+      # Window dimensions
+      params[:window_height] = options[:window_height] if options[:window_height]
+      params[:window_width] = options[:window_width] if options[:window_width]
+
+      # Screenshots
+      params[:screenshot] = "true" if options[:screenshot]
+      params[:screenshot_fullpage] = "true" if options[:screenshot_fullpage]
+      params[:screenshot_selector] = options[:screenshot_selector] if options[:screenshot_selector]
+
+      # JS instructions
+      if options[:js_instructions]
+        instructions = options[:js_instructions]
+        instructions = instructions.to_json if instructions.respond_to?(:to_a)
+        params[:js_instructions] = instructions
+      end
+
+      # Other options
+      params[:block_resources] = options[:block_resources] if options[:block_resources]
+      params[:device] = options[:device] if options[:device]
+      params[:antibot] = "true" if options[:antibot]
+
+      # Custom headers
+      options[:headers]&.each do |key, value|
+        params["custom_headers[#{key}]"] = value
+      end
+
+      params
+    end
+
+    def handle_response(http_response, options)
+      case http_response.status.code
+      when 200..299
+        ApiResponse.new(http_response, options)
+      when 401
+        raise AuthenticationError, "Invalid API key"
+      when 429
+        retry_after = http_response.headers["Retry-After"]&.to_i
+        raise RateLimitError.new("Rate limited", retry_after: retry_after)
+      when 403
+        raise BotDetectedError.new("Bot detected", suggestion: "Try premium_proxy or antibot options")
+      else
+        body = http_response.body.to_s
+        raise Error, "API error (#{http_response.status}): #{body[0, 200]}"
+      end
+    end
+  end
+end

data/lib/zenrows/api_response.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Zenrows
+  # Response wrapper for ZenRows API responses
+  #
+  # Provides convenient accessors for different response types based on
+  # the options used in the request.
+  #
+  # @example HTML response
+  #   response = api.get(url)
+  #   response.html  # => "<html>..."
+  #
+  # @example JSON response with XHR data
+  #   response = api.get(url, json_response: true)
+  #   response.data  # => { "html" => "...", "xhr" => [...] }
+  #   response.html  # => "<html>..."
+  #   response.xhr   # => [...]
+  #
+  # @example Autoparse response
+  #   response = api.get(url, autoparse: true)
+  #   response.parsed  # => { "title" => "...", "price" => "..." }
+  #
+  # @example CSS extraction
+  #   response = api.get(url, css_extractor: { title: 'h1' })
+  #   response.extracted  # => { "title" => "Page Title" }
+  #
+  # @example Markdown response
+  #   response = api.get(url, response_type: 'markdown')
+  #   response.markdown  # => "# Page Title\n\n..."
+  #
+  # @author Ernest Bursa
+  # @since 0.2.0
+  # @api public
+  class ApiResponse
+    # @return [HTTP::Response] Raw HTTP response
+    attr_reader :raw
+
+    # @return [Integer] HTTP status code
+    attr_reader :status
+
+    # @return [Hash] Request options used
+    attr_reader :options
+
+    # Initialize response wrapper
+    #
+    # @param http_response [HTTP::Response] Raw HTTP response
+    # @param options [Hash] Request options
+    def initialize(http_response, options = {})
+      @raw = http_response
+      @status = http_response.status.code
+      @options = options
+      @body = http_response.body.to_s
+      @parsed_json = nil
+    end
+
+    # Response body as string
+    #
+    # @return [String] Raw response body
+    attr_reader :body
+
+    # Parsed data (for JSON responses)
+    #
+    # @return [Hash, Array, String] Parsed response data
+    def data
+      @parsed_json ||= parse_body
+    end
+
+    # HTML content
+    #
+    # Returns HTML from json_response data or raw body
+    #
+    # @return [String] HTML content
+    def html
+      if json_response?
+        data.is_a?(Hash) ? data["html"] : data
+      else
+        @body
+      end
+    end
+
+    # Markdown content (when response_type: 'markdown')
+    #
+    # @return [String] Markdown content
+    def markdown
+      @body
+    end
+
+    # Parsed/extracted data (for autoparse or css_extractor)
+    #
+    # @return [Hash] Structured data
+    def parsed
+      data
+    end
+
+    # Alias for parsed data when using css_extractor
+    #
+    # @return [Hash] Extracted data
+    def extracted
+      data
+    end
+
+    # XHR/fetch request data (when json_response: true)
+    #
+    # @return [Array, nil] XHR request data
+    def xhr
+      data.is_a?(Hash) ? data["xhr"] : nil
+    end
+
+    # JS instructions execution report (when json_response: true)
+    #
+    # @return [Hash, nil] Instructions report
+    def js_instructions_report
+      data.is_a?(Hash) ? data["js_instructions_report"] : nil
+    end
+
+    # Screenshot data (when screenshot options used with json_response)
+    #
+    # @return [String, nil] Base64 encoded screenshot
+    def screenshot
+      data.is_a?(Hash) ? data["screenshot"] : nil
+    end
+
+    # Response headers
+    #
+    # @return [Hash] Response headers
+    def headers
+      @raw.headers.to_h
+    end
+
+    # Concurrency limit from headers
+    #
+    # @return [Integer, nil] Max concurrent requests
+    def concurrency_limit
+      headers["Concurrency-Limit"]&.to_i
+    end
+
+    # Remaining concurrency from headers
+    #
+    # @return [Integer, nil] Available concurrent request slots
+    def concurrency_remaining
+      headers["Concurrency-Remaining"]&.to_i
+    end
+
+    # Request cost from headers
+    #
+    # @return [Float, nil] Credit cost of request
+    def request_cost
+      headers["X-Request-Cost"]&.to_f
+    end
+
+    # Final URL after redirects
+    #
+    # @return [String, nil] Final URL
+    def final_url
+      headers["Zr-Final-Url"]
+    end
+
+    # Check if response is successful
+    #
+    # @return [Boolean]
+    def success?
+      status >= 200 && status < 300
+    end
+
+    private
+
+    def json_response?
+      options[:json_response] || options[:autoparse] || options[:css_extractor] || options[:outputs]
+    end
+
+    def parse_body
+      return @body unless json_response? || looks_like_json?
+
+      JSON.parse(@body)
+    rescue JSON::ParserError
+      @body
+    end
+
+    def looks_like_json?
+      @body.start_with?("{") || @body.start_with?("[")
+    end
+  end
+end

data/lib/zenrows/backends/http_rb.rb

@@ -12,7 +12,7 @@ module Zenrows
     # @example Basic usage
     #   backend = Zenrows::Backends::HttpRb.new(proxy: proxy, config: config)
     #   http = backend.build_client(js_render: true)
-    #   response = http.get(url, ssl_context: backend.ssl_context)
+    #   response = http.get(url)  # SSL context is auto-configured
     #
     # @author Ernest Bursa
     # @since 0.1.0
@@ -41,18 +41,17 @@ module Zenrows
         # Calculate timeouts
         timeouts = calculate_timeouts(opts)
 
-        # Build HTTP client
-        client = HTTP
+        # Build HTTP client with SSL context and proxy
+        HTTP
           .timeout(connect: timeouts[:connect], read: timeouts[:read])
           .headers(headers)
-
-        # Configure proxy
-        client.via(
-          proxy_config[:host],
-          proxy_config[:port],
-          proxy_config[:username],
-          proxy_config[:password]
-        )
+          .via(
+            proxy_config[:host],
+            proxy_config[:port],
+            proxy_config[:username],
+            proxy_config[:password],
+            ssl_context: ssl_context
+          )
       end
     end
   end