zenrows 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af4d111e269c080e71f047c74acec3a319f05e0b6c800cb7067b4d05a23cc209
-  data.tar.gz: 373c8b95c4f311141767d750a509e88ef74dc2e18c947a703b9868c5cce97ad4
+  metadata.gz: e684c8840821205883d52bd782aa3d22c8c21404bc9e3a4ab068f9069389e575
+  data.tar.gz: eed9546964086c061082a55aa4fa9218c5c24dfa02b70f20297b04b55fa77891
 SHA512:
-  metadata.gz: 81b92b61f13e82a6b2462e790df7f009f88d75f7d9cac822dfd5e685fab0c85b85898f50f3ce67b241622609ff02ebf2b98896e0b9a2ed870f41d1ea2231138c
-  data.tar.gz: 9740ef08c21c4d3701795786886df16183d188a0cf6d5fc7d04bc0a2a46b8f00693ec98fecfd4925bf9a2cd69ab9dca93add9e4c54f2631c630f2698444f912a
+  metadata.gz: 31a6e6f8d95e0431cd7ce0bf545e813c64954021914eb67cf8312155b0c9d7cdea2ab1ede5378972757c1b148be0cd7be13e7ddda5ed2fbdb83cb30aad9fc18a
+  data.tar.gz: 74584d555a2915b1e0f8ac2fd0fb46ca78b21a80430c0e20f252237497a27087b4027e244c450222ae322b2f1ae2f0a02b59652daddc2069ac48783525d5cc69

data/.mcp.json

@@ -0,0 +1,10 @@
+{
+  "mcpServers": {
+    "Context7": {
+      "type": "stdio",
+      "command": "npx",
+      "args": ["-y", "@upstash/context7-mcp@latest"],
+      "env": {}
+    }
+  }
+}

data/.tool-versions

@@ -0,0 +1 @@
+ruby 3.4.2

data/CHANGELOG.md

@@ -7,6 +7,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.0] - 2025-12-30
+
+### Added
+
+- Hooks/callbacks system for request lifecycle events
+- Five hook types: `before_request`, `after_request`, `on_response`, `on_error`, `around_request`
+- Global and per-client hook registration
+- `Zenrows::Hooks::LogSubscriber` built-in logging subscriber
+- `InstrumentedClient` wrapper for HTTP client instrumentation
+- Context object with ZenRows header parsing (request_cost, concurrency_remaining, request_id, final_url)
+- RBS type signatures for all hooks classes
+- Monotonic clock for accurate request duration timing
+
+### Changed
+
+- `after_request` hook now always runs (via `ensure`), even on errors
+
+## [0.2.1] - 2025-12-25
+
+### Added
+
+- Configurable `api_endpoint` for ApiClient (global config or per-instance)
+- `net_http` backend as fallback when http.rb unavailable
+- GitHub Pages for YARD documentation
+
+### Changed
+
+- SSL context now auto-configured in proxy client (no need to pass `ssl_context:` on every request)
+
+## [0.2.0] - 2025-12-25
+
+### Added
+
+- `ApiClient` for REST API mode (autoparse, css_extractor, markdown output)
+- `CssExtractor` DSL for building extraction rules
+- `ApiResponse` wrapper with typed accessors
+- Proxy options: `device`, `antibot`, `session_ttl`
+- RBS type signatures
+- rubocop-minitest linting
+- Dependabot configuration
+
+### Changed
+
+- Require Ruby >= 3.2.0
+
 ## [0.1.0] - 2025-12-25
 
 ### Added

data/CLAUDE.md

@@ -46,7 +46,7 @@ end
 
 client = Zenrows::Client.new
 http = client.http(js_render: true, premium_proxy: true)
-response = http.get('https://example.com', ssl_context: client.ssl_context)
+response = http.get('https://example.com')  # SSL verification disabled for proxy
 ```
 
 ## Key Options

data/Makefile

@@ -0,0 +1,19 @@
+.PHONY: install test lint rbs docs build
+
+install:
+	bundle install
+
+test:
+	bundle exec rake test
+
+lint:
+	bundle exec rubocop
+
+rbs:
+	rbs -I sig -r monitor -r openssl -r logger validate
+
+docs:
+	bundle exec yard doc
+
+build:
+	bundle exec rake build

data/README.md

@@ -35,12 +35,14 @@ end
 ```ruby
 client = Zenrows::Client.new
 http = client.http(js_render: true, premium_proxy: true)
-response = http.get('https://example.com', ssl_context: client.ssl_context)
+response = http.get('https://example.com')
 
 puts response.body
 puts response.status
 ```
 
+> **Note:** SSL verification is disabled automatically for proxy connections (required by ZenRows).
+
 ### With Options
 
 ```ruby
@@ -70,10 +72,11 @@ instructions = Zenrows::JsInstructions.build do
 end
 
 http = client.http(js_render: true, js_instructions: instructions)
-response = http.get(url, ssl_context: client.ssl_context)
+response = http.get(url)
 ```
 
 Available instructions:
+
 - `click(selector)` - Click element
 - `wait(ms)` - Wait duration
 - `wait_for(selector)` - Wait for element
@@ -108,25 +111,142 @@ http = client.http(
 )
 ```
 
+### Device & Antibot
+
+```ruby
+http = client.http(
+  js_render: true,
+  device: 'mobile',      # mobile/desktop emulation
+  antibot: true          # enhanced antibot bypass
+)
+```
+
+## API Client (v0.2.0+)
+
+For advanced extraction features, use the REST API client:
+
+### Autoparse
+
+Extract structured data from known sites (Amazon, etc.):
+
+```ruby
+api = Zenrows::ApiClient.new
+response = api.get('https://amazon.com/dp/B01LD5GO7I', autoparse: true)
+
+response.parsed  # => { "title" => "...", "price" => "$29.99", ... }
+```
+
+### CSS Extraction
+
+Extract data using CSS selectors:
+
+```ruby
+# Hash syntax
+response = api.get(url, css_extractor: {
+  title: 'h1',
+  links: 'a @href',
+  prices: '.price'
+})
+response.extracted  # => { "title" => "...", "links" => [...], "prices" => [...] }
+
+# DSL syntax
+extractor = Zenrows::CssExtractor.build do
+  extract :title, 'h1'
+  links :urls, 'a.product'
+  images :photos, 'img.gallery'
+end
+response = api.get(url, css_extractor: extractor)
+```
+
+### Markdown Output
+
+```ruby
+response = api.get(url, response_type: 'markdown')
+response.markdown  # => "# Page Title\n\nContent..."
+```
+
+### Response Metadata
+
+```ruby
+response = api.get(url)
+response.status              # => 200
+response.success?            # => true
+response.final_url           # => "https://example.com/redirected"
+response.request_cost        # => 0.001
+response.concurrency_remaining  # => 199
+```
+
 ## Options Reference
 
-| Option | Type | Description |
-|--------|------|-------------|
-| `js_render` | Boolean | Enable JavaScript rendering |
-| `premium_proxy` | Boolean | Use residential proxies |
-| `proxy_country` | String | Country code (us, gb, de, etc.) |
-| `wait` | Integer/Boolean | Wait time in ms (true = 15000) |
-| `wait_for` | String | CSS selector to wait for |
-| `session_id` | Boolean/String | Session persistence |
-| `window_height` | Integer | Browser window height |
-| `window_width` | Integer | Browser window width |
-| `js_instructions` | Array/String | Browser automation |
-| `json_response` | Boolean | Return JSON instead of HTML |
-| `screenshot` | Boolean | Take screenshot |
-| `screenshot_fullpage` | Boolean | Full page screenshot |
-| `screenshot_selector` | String | Screenshot specific element |
-| `block_resources` | String | Block resources (image,media,font) |
-| `headers` | Hash | Custom HTTP headers |
+| Option                | Type            | Description                        |
+| --------------------- | --------------- | ---------------------------------- |
+| `js_render`           | Boolean         | Enable JavaScript rendering        |
+| `premium_proxy`       | Boolean         | Use residential proxies            |
+| `proxy_country`       | String          | Country code (us, gb, de, etc.)    |
+| `device`              | String          | Device emulation (mobile/desktop)  |
+| `antibot`             | Boolean         | Enhanced antibot bypass            |
+| `wait`                | Integer/Boolean | Wait time in ms (true = 15000)     |
+| `wait_for`            | String          | CSS selector to wait for           |
+| `session_id`          | Boolean/String  | Session persistence                |
+| `session_ttl`         | String          | Session duration (1m, 10m, 30m)    |
+| `window_height`       | Integer         | Browser window height              |
+| `window_width`        | Integer         | Browser window width               |
+| `js_instructions`     | Array/String    | Browser automation                 |
+| `json_response`       | Boolean         | Return JSON instead of HTML        |
+| `screenshot`          | Boolean         | Take screenshot                    |
+| `screenshot_fullpage` | Boolean         | Full page screenshot               |
+| `screenshot_selector` | String          | Screenshot specific element        |
+| `block_resources`     | String          | Block resources (image,media,font) |
+| `headers`             | Hash            | Custom HTTP headers                |
+
+### API Client Options
+
+| Option           | Type         | Description                          |
+| ---------------- | ------------ | ------------------------------------ |
+| `autoparse`      | Boolean      | Auto-extract structured data         |
+| `css_extractor`  | Hash/Object  | CSS selectors for extraction         |
+| `response_type`  | String       | Output format ('markdown')           |
+| `outputs`        | String       | Extract specific data (headings,links) |
+
+## Hooks
+
+Register callbacks for request lifecycle events:
+
+```ruby
+Zenrows.configure do |c|
+  c.api_key = 'YOUR_KEY'
+
+  # Log responses
+  c.on_response { |resp, ctx| puts "#{ctx[:host]} -> #{resp.status}" }
+
+  # Track errors
+  c.on_error { |err, ctx| Sentry.capture_exception(err) }
+
+  # Monitor costs
+  c.on_response do |resp, ctx|
+    cost = ctx[:zenrows_headers][:request_cost]
+    StatsD.increment('zenrows.cost', cost) if cost
+  end
+end
+```
+
+Per-client hooks:
+
+```ruby
+client = Zenrows::Client.new do |c|
+  c.on_response { |resp, ctx| log_specific(resp) }
+end
+```
+
+Built-in logger:
+
+```ruby
+c.add_subscriber(Zenrows::Hooks::LogSubscriber.new)
+```
+
+Available hooks: `before_request`, `after_request`, `on_response`, `on_error`, `around_request`
+
+Context includes: `method`, `url`, `host`, `duration`, `zenrows_headers` (request_cost, concurrency_remaining, request_id, final_url)
 
 ## Error Handling
 
@@ -152,6 +272,7 @@ end
 ## Rails Integration
 
 The gem automatically integrates with Rails when detected:
+
 - Uses Rails.logger by default
 - Supports ActiveSupport::Duration for wait times
 

data/lib/zenrows/api_client.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require "http"
+require "json"
+require "cgi"
+
+module Zenrows
+  # REST API client for ZenRows Universal Scraper API
+  #
+  # Unlike the proxy-based Client, ApiClient calls the ZenRows API directly.
+  # This enables features not available in proxy mode: autoparse, css_extractor,
+  # response_type (markdown), and outputs.
+  #
+  # @example Basic usage
+  #   api = Zenrows::ApiClient.new
+  #   response = api.get('https://example.com')
+  #
+  # @example With autoparse
+  #   response = api.get('https://amazon.com/dp/B01LD5GO7I', autoparse: true)
+  #   puts response.data  # Structured product data
+  #
+  # @example With CSS extraction
+  #   response = api.get(url, css_extractor: { title: 'h1', links: 'a @href' })
+  #
+  # @example With markdown output
+  #   response = api.get(url, response_type: 'markdown')
+  #
+  # @example With per-client hooks
+  #   api = Zenrows::ApiClient.new do |c|
+  #     c.on_response { |resp, ctx| puts "#{ctx[:host]} -> #{resp.status}" }
+  #   end
+  #
+  # @author Ernest Bursa
+  # @since 0.2.0
+  # @api public
+  class ApiClient
+    # @return [String] ZenRows API key
+    attr_reader :api_key
+
+    # @return [String] API endpoint URL
+    attr_reader :api_endpoint
+
+    # @return [Configuration] Configuration instance
+    attr_reader :config
+
+    # @return [Hooks] Hook registry for this client
+    attr_reader :hooks
+
+    # Initialize API client
+    #
+    # @param api_key [String, nil] Override API key (uses global config if nil)
+    # @param api_endpoint [String, nil] Override API endpoint (uses global config if nil)
+    # @yield [config] Optional block for per-client configuration (hooks)
+    # @yieldparam config [HookConfigurator] Hook configuration DSL
+    def initialize(api_key: nil, api_endpoint: nil, &block)
+      @config = Zenrows.configuration
+      @api_key = api_key || @config.api_key
+      @api_endpoint = api_endpoint || @config.api_endpoint
+      @config.validate! unless api_key
+
+      # Build hooks: start with global, allow per-client additions
+      @hooks = block ? build_hooks(&block) : Zenrows.configuration.hooks.dup
+    end
+
+    # Make GET request through ZenRows API
+    #
+    # @param url [String] Target URL to scrape
+    # @param options [Hash] Request options
+    # @option options [Boolean] :autoparse Auto-extract structured data
+    # @option options [Hash, CssExtractor] :css_extractor CSS selectors for extraction
+    # @option options [String] :response_type Response format ('markdown')
+    # @option options [String] :outputs Extract specific data ('headings,links,menus')
+    # @option options [Boolean] :js_render Enable JavaScript rendering
+    # @option options [Boolean] :premium_proxy Use residential proxies
+    # @option options [String] :proxy_country Country code
+    # @option options [Integer, Boolean] :wait Wait time in ms
+    # @option options [String] :wait_for CSS selector to wait for
+    # @option options [Boolean, String] :session_id Session persistence
+    # @option options [Array, String] :js_instructions Browser automation
+    # @option options [Boolean] :json_response Return JSON with XHR data
+    # @option options [Boolean] :screenshot Take screenshot
+    # @option options [Boolean] :screenshot_fullpage Full page screenshot
+    # @option options [String] :screenshot_selector Screenshot element
+    # @option options [String] :block_resources Block resources
+    # @option options [String] :device Device emulation
+    # @option options [Boolean] :antibot Enhanced antibot
+    # @return [ApiResponse] Response wrapper
+    # @raise [ConfigurationError] if API key not configured
+    # @raise [AuthenticationError] if API key invalid
+    # @raise [RateLimitError] if rate limited
+    def get(url, **options)
+      instrument(:get, url, options) do
+        params = build_params(url, options)
+        http_response = build_http_client.get(api_endpoint, params: params)
+        handle_response(http_response, options)
+      end
+    end
+
+    # Make POST request through ZenRows API
+    #
+    # @param url [String] Target URL
+    # @param body [String, Hash] Request body
+    # @param options [Hash] Request options (same as #get)
+    # @return [ApiResponse] Response wrapper
+    def post(url, body: nil, **options)
+      instrument(:post, url, options) do
+        params = build_params(url, options)
+        http_response = build_http_client.post(api_endpoint, params: params, body: body)
+        handle_response(http_response, options)
+      end
+    end
+
+    private
+
+    # Build hooks registry for this client
+    #
+    # @yield [config] Block for registering per-client hooks
+    # @return [Hooks] Combined hooks registry
+    def build_hooks
+      client_hooks = Zenrows.configuration.hooks.dup
+      hook_config = HookConfigurator.new(client_hooks)
+      yield(hook_config)
+      client_hooks
+    end
+
+    # Instrument a request with hooks
+    #
+    # @param method [Symbol] HTTP method
+    # @param url [String] Target URL
+    # @param options [Hash] Request options
+    # @yield Block that executes the actual request
+    # @return [Object] Response from block
+    def instrument(method, url, options)
+      return yield if hooks.empty?
+
+      context = Hooks::Context.for_request(
+        method: method,
+        url: url,
+        options: options,
+        backend: :api
+      )
+
+      hooks.run(:before_request, context)
+
+      response = hooks.run_around(context) do
+        result = yield
+        Hooks::Context.enrich_with_response(context, result)
+        hooks.run(:on_response, result, context)
+        result
+      end
+
+      response
+    rescue => e
+      context[:error] = e if context
+      hooks.run(:on_error, e, context) if context
+      raise
+    ensure
+      hooks.run(:after_request, context) if context
+    end
+
+    def build_http_client
+      HTTP
+        .timeout(connect: config.connect_timeout, read: config.read_timeout)
+    end
+
+    def build_params(url, options)
+      params = {apikey: api_key, url: url}
+
+      # API-mode only features
+      params[:autoparse] = "true" if options[:autoparse]
+      params[:response_type] = options[:response_type] if options[:response_type]
+      params[:outputs] = options[:outputs] if options[:outputs]
+
+      if options[:css_extractor]
+        extractor = options[:css_extractor]
+        params[:css_extractor] = extractor.to_json
+      end
+
+      # Common options (also available in proxy mode)
+      params[:js_render] = "true" if options[:js_render]
+      params[:premium_proxy] = "true" if options[:premium_proxy]
+      params[:proxy_country] = options[:proxy_country] if options[:proxy_country]
+      params[:json_response] = "true" if options[:json_response]
+      params[:original_status] = "true" if options[:original_status]
+
+      # Wait options
+      if options[:wait]
+        params[:wait] = (options[:wait] == true) ? 15000 : options[:wait]
+      end
+      params[:wait_for] = options[:wait_for] if options[:wait_for]
+
+      # Session
+      if options[:session_id]
+        params[:session_id] = (options[:session_id] == true) ? rand(1..99999) : options[:session_id]
+      end
+
+      # Window dimensions
+      params[:window_height] = options[:window_height] if options[:window_height]
+      params[:window_width] = options[:window_width] if options[:window_width]
+
+      # Screenshots
+      params[:screenshot] = "true" if options[:screenshot]
+      params[:screenshot_fullpage] = "true" if options[:screenshot_fullpage]
+      params[:screenshot_selector] = options[:screenshot_selector] if options[:screenshot_selector]
+
+      # JS instructions
+      if options[:js_instructions]
+        instructions = options[:js_instructions]
+        instructions = instructions.to_json if instructions.respond_to?(:to_a)
+        params[:js_instructions] = instructions
+      end
+
+      # Other options
+      params[:block_resources] = options[:block_resources] if options[:block_resources]
+      params[:device] = options[:device] if options[:device]
+      params[:antibot] = "true" if options[:antibot]
+
+      # Custom headers
+      options[:headers]&.each do |key, value|
+        params["custom_headers[#{key}]"] = value
+      end
+
+      params
+    end
+
+    def handle_response(http_response, options)
+      case http_response.status.code
+      when 200..299
+        ApiResponse.new(http_response, options)
+      when 401
+        raise AuthenticationError, "Invalid API key"
+      when 429
+        retry_after = http_response.headers["Retry-After"]&.to_i
+        raise RateLimitError.new("Rate limited", retry_after: retry_after)
+      when 403
+        raise BotDetectedError.new("Bot detected", suggestion: "Try premium_proxy or antibot options")
+      else
+        body = http_response.body.to_s
+        raise Error, "API error (#{http_response.status}): #{body[0, 200]}"
+      end
+    end
+  end
+end