RubyGems - zenrows - Versions diffs - 0.1.0 → 0.3.0 - Mend

zenrows 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (54) hide show

checksums.yaml +4 -4
data/.mcp.json +10 -0
data/.tool-versions +1 -0
data/CHANGELOG.md +45 -0
data/CLAUDE.md +1 -1
data/Makefile +19 -0
data/README.md +140 -19
data/lib/zenrows/api_client.rb +243 -0
data/lib/zenrows/api_response.rb +185 -0
data/lib/zenrows/backends/base.rb +31 -1
data/lib/zenrows/backends/http_rb.rb +17 -10
data/lib/zenrows/backends/net_http.rb +149 -0
data/lib/zenrows/client.rb +120 -11
data/lib/zenrows/configuration.rb +117 -0
data/lib/zenrows/css_extractor.rb +111 -0
data/lib/zenrows/hooks/context.rb +142 -0
data/lib/zenrows/hooks/log_subscriber.rb +124 -0
data/lib/zenrows/hooks.rb +213 -0
data/lib/zenrows/instrumented_client.rb +187 -0
data/lib/zenrows/proxy.rb +19 -0
data/lib/zenrows/version.rb +1 -1
data/lib/zenrows.rb +14 -2
data/sig/manifest.yaml +5 -0
data/sig/zenrows/api_client.rbs +18 -0
data/sig/zenrows/api_response.rbs +28 -0
data/sig/zenrows/backends/base.rbs +12 -0
data/sig/zenrows/backends/http_rb.rbs +3 -0
data/sig/zenrows/backends/net_http.rbs +28 -0
data/sig/zenrows/backends.rbs +2 -0
data/sig/zenrows/client.rbs +12 -0
data/sig/zenrows/configuration.rbs +29 -0
data/sig/zenrows/css_extractor.rbs +14 -0
data/sig/zenrows/errors.rbs +27 -0
data/sig/zenrows/hook_configurator.rbs +9 -0
data/sig/zenrows/hooks/context.rbs +6 -0
data/sig/zenrows/hooks/log_subscriber.rbs +15 -0
data/sig/zenrows/hooks.rbs +23 -0
data/sig/zenrows/instrumented_client.rbs +22 -0
data/sig/zenrows/js_instructions.rbs +28 -0
data/sig/zenrows/proxy.rbs +14 -0
data/sig/zenrows.rbs +4 -1
data/test/test_helper.rb +42 -0
data/test/zenrows/api_client_test.rb +161 -0
data/test/zenrows/api_response_test.rb +142 -0
data/test/zenrows/client_hooks_test.rb +105 -0
data/test/zenrows/configuration_hooks_test.rb +101 -0
data/test/zenrows/css_extractor_test.rb +84 -0
data/test/zenrows/hooks/context_test.rb +150 -0
data/test/zenrows/hooks/log_subscriber_test.rb +105 -0
data/test/zenrows/hooks_test.rb +215 -0
data/test/zenrows/instrumented_client_test.rb +153 -0
data/test/zenrows/js_instructions_test.rb +2 -1
data/test/zenrows/proxy_test.rb +39 -0
metadata +42 -4

data/lib/zenrows/api_response.rb ADDED Viewed

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+require "json"
+module Zenrows
+  # Response wrapper for ZenRows API responses
+  #
+  # Provides convenient accessors for different response types based on
+  # the options used in the request.
+  #
+  # @example HTML response
+  #   response = api.get(url)
+  #   response.html  # => "<html>..."
+  #
+  # @example JSON response with XHR data
+  #   response = api.get(url, json_response: true)
+  #   response.data  # => { "html" => "...", "xhr" => [...] }
+  #   response.html  # => "<html>..."
+  #   response.xhr   # => [...]
+  #
+  # @example Autoparse response
+  #   response = api.get(url, autoparse: true)
+  #   response.parsed  # => { "title" => "...", "price" => "..." }
+  #
+  # @example CSS extraction
+  #   response = api.get(url, css_extractor: { title: 'h1' })
+  #   response.extracted  # => { "title" => "Page Title" }
+  #
+  # @example Markdown response
+  #   response = api.get(url, response_type: 'markdown')
+  #   response.markdown  # => "# Page Title\n\n..."
+  #
+  # @author Ernest Bursa
+  # @since 0.2.0
+  # @api public
+  class ApiResponse
+    # @return [HTTP::Response] Raw HTTP response
+    attr_reader :raw
+    # @return [Integer] HTTP status code
+    attr_reader :status
+    # @return [Hash] Request options used
+    attr_reader :options
+    # Initialize response wrapper
+    #
+    # @param http_response [HTTP::Response] Raw HTTP response
+    # @param options [Hash] Request options
+    def initialize(http_response, options = {})
+      @raw = http_response
+      @status = http_response.status.code
+      @options = options
+      @body = http_response.body.to_s
+      @parsed_json = nil
+    end
+    # Response body as string
+    #
+    # @return [String] Raw response body
+    attr_reader :body
+    # Parsed data (for JSON responses)
+    #
+    # @return [Hash, Array, String] Parsed response data
+    def data
+      @parsed_json ||= parse_body
+    end
+    # HTML content
+    #
+    # Returns HTML from json_response data or raw body
+    #
+    # @return [String] HTML content
+    def html
+      if json_response?
+        data.is_a?(Hash) ? data["html"] : data
+      else
+        @body
+      end
+    end
+    # Markdown content (when response_type: 'markdown')
+    #
+    # @return [String] Markdown content
+    def markdown
+      @body
+    end
+    # Parsed/extracted data (for autoparse or css_extractor)
+    #
+    # @return [Hash] Structured data
+    def parsed
+      data
+    end
+    # Alias for parsed data when using css_extractor
+    #
+    # @return [Hash] Extracted data
+    def extracted
+      data
+    end
+    # XHR/fetch request data (when json_response: true)
+    #
+    # @return [Array, nil] XHR request data
+    def xhr
+      data.is_a?(Hash) ? data["xhr"] : nil
+    end
+    # JS instructions execution report (when json_response: true)
+    #
+    # @return [Hash, nil] Instructions report
+    def js_instructions_report
+      data.is_a?(Hash) ? data["js_instructions_report"] : nil
+    end
+    # Screenshot data (when screenshot options used with json_response)
+    #
+    # @return [String, nil] Base64 encoded screenshot
+    def screenshot
+      data.is_a?(Hash) ? data["screenshot"] : nil
+    end
+    # Response headers
+    #
+    # @return [Hash] Response headers
+    def headers
+      @raw.headers.to_h
+    end
+    # Concurrency limit from headers
+    #
+    # @return [Integer, nil] Max concurrent requests
+    def concurrency_limit
+      headers["Concurrency-Limit"]&.to_i
+    end
+    # Remaining concurrency from headers
+    #
+    # @return [Integer, nil] Available concurrent request slots
+    def concurrency_remaining
+      headers["Concurrency-Remaining"]&.to_i
+    end
+    # Request cost from headers
+    #
+    # @return [Float, nil] Credit cost of request
+    def request_cost
+      headers["X-Request-Cost"]&.to_f
+    end
+    # Final URL after redirects
+    #
+    # @return [String, nil] Final URL
+    def final_url
+      headers["Zr-Final-Url"]
+    end
+    # Check if response is successful
+    #
+    # @return [Boolean]
+    def success?
+      status >= 200 && status < 300
+    end
+    private
+    def json_response?
+      options[:json_response] || options[:autoparse] || options[:css_extractor] || options[:outputs]
+    end
+    def parse_body
+      return @body unless json_response? || looks_like_json?
+      JSON.parse(@body)
+    rescue JSON::ParserError
+      @body
+    end
+    def looks_like_json?
+      @body.start_with?("{") || @body.start_with?("[")
+    end
+  end
+end

data/lib/zenrows/backends/base.rb CHANGED Viewed

@@ -18,11 +18,16 @@ module Zenrows
       # @return [Zenrows::Configuration] Configuration instance
       attr_reader :config
+      # @return [Zenrows::Hooks] Hook registry for this backend
+      attr_reader :hooks
       # @param proxy [Zenrows::Proxy] Proxy configuration builder
       # @param config [Zenrows::Configuration] Configuration instance
-      def initialize(proxy:, config:)
+      # @param hooks [Zenrows::Hooks, nil] Optional hook registry (defaults to config.hooks)
+      def initialize(proxy:, config:, hooks: nil)
         @proxy = proxy
         @config = config
+        @hooks = hooks || config.hooks&.dup || Hooks.new
       end
       # Build a configured HTTP client
@@ -74,6 +79,31 @@ module Zenrows
         {connect: connect, read: read}
       end
+      # Wrap HTTP client with instrumentation if hooks are registered
+      #
+      # @param client [Object] The underlying HTTP client
+      # @param options [Hash] Request options used for this client
+      # @return [Object] Instrumented client or original if no hooks
+      def wrap_client(client, options)
+        return client if hooks.empty?
+        InstrumentedClient.new(
+          client,
+          hooks: hooks,
+          context_base: {
+            options: options,
+            backend: backend_name
+          }
+        )
+      end
+      # Get the backend name for context
+      #
+      # @return [Symbol] Backend identifier
+      def backend_name
+        :base
+      end
       private
       # Normalize wait value to seconds

data/lib/zenrows/backends/http_rb.rb CHANGED Viewed

@@ -12,7 +12,7 @@ module Zenrows
     # @example Basic usage
     #   backend = Zenrows::Backends::HttpRb.new(proxy: proxy, config: config)
     #   http = backend.build_client(js_render: true)
-    #   response = http.get(url, ssl_context: backend.ssl_context)
+    #   response = http.get(url)  # SSL context is auto-configured
     #
     # @author Ernest Bursa
     # @since 0.1.0
@@ -27,7 +27,7 @@ module Zenrows
       # @option options [Boolean, Integer] :wait Wait time
       # @option options [String] :wait_for CSS selector to wait for
       # @option options [Hash] :headers Custom HTTP headers
-      # @return [HTTP::Client] Configured HTTP client
+      # @return [HTTP::Client, InstrumentedClient] Configured HTTP client (instrumented if hooks registered)
       def build_client(options = {})
         opts = options.dup
         headers = opts.delete(:headers) || {}
@@ -41,18 +41,25 @@ module Zenrows
         # Calculate timeouts
         timeouts = calculate_timeouts(opts)
-        # Build HTTP client
+        # Build HTTP client with SSL context and proxy
         client = HTTP
           .timeout(connect: timeouts[:connect], read: timeouts[:read])
           .headers(headers)
+          .via(
+            proxy_config[:host],
+            proxy_config[:port],
+            proxy_config[:username],
+            proxy_config[:password],
+            ssl_context: ssl_context
+          )
-        # Configure proxy
-        client.via(
-          proxy_config[:host],
-          proxy_config[:port],
-          proxy_config[:username],
-          proxy_config[:password]
-        )
+        # Wrap with instrumentation if hooks registered
+        wrap_client(client, opts)
+      end
+      # @return [Symbol] Backend identifier
+      def backend_name
+        :http_rb
       end
     end
   end

data/lib/zenrows/backends/net_http.rb ADDED Viewed

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+require "net/http"
+require "uri"
+require "openssl"
+module Zenrows
+  module Backends
+    # Net::HTTP backend adapter (stdlib fallback)
+    #
+    # Uses Ruby's built-in Net::HTTP when http.rb is not available.
+    # Provides basic proxy support with SSL verification disabled.
+    #
+    # @example Basic usage
+    #   backend = Zenrows::Backends::NetHttp.new(proxy: proxy, config: config)
+    #   http = backend.build_client(js_render: true)
+    #   response = http.get(url)
+    #
+    # @author Ernest Bursa
+    # @since 0.2.1
+    # @api public
+    class NetHttp < Base
+      # Build a configured HTTP client wrapper
+      #
+      # @param options [Hash] Request options
+      # @return [NetHttpClient, InstrumentedClient] Configured client wrapper (instrumented if hooks registered)
+      def build_client(options = {})
+        opts = options.dup
+        headers = opts.delete(:headers) || {}
+        opts[:custom_headers] = true if headers.any?
+        proxy_config = proxy.build(opts)
+        timeouts = calculate_timeouts(opts)
+        client = NetHttpClient.new(
+          proxy_config: proxy_config,
+          headers: headers,
+          timeouts: timeouts,
+          ssl_context: ssl_context
+        )
+        # Wrap with instrumentation if hooks registered
+        wrap_client(client, opts)
+      end
+      # @return [Symbol] Backend identifier
+      def backend_name
+        :net_http
+      end
+    end
+    # Wrapper around Net::HTTP that mimics http.rb interface
+    #
+    # @api private
+    class NetHttpClient
+      # @param proxy_config [Hash] Proxy configuration
+      # @param headers [Hash] Default headers
+      # @param timeouts [Hash] Timeout configuration
+      # @param ssl_context [OpenSSL::SSL::SSLContext] SSL context
+      def initialize(proxy_config:, headers:, timeouts:, ssl_context:)
+        @proxy_config = proxy_config
+        @headers = headers
+        @timeouts = timeouts
+        @ssl_context = ssl_context
+      end
+      # Make GET request
+      #
+      # @param url [String] Target URL
+      # @param options [Hash] Request options
+      # @return [NetHttpResponse] Response wrapper
+      def get(url, **options)
+        uri = URI.parse(url)
+        request(uri, Net::HTTP::Get.new(uri), options)
+      end
+      # Make POST request
+      #
+      # @param url [String] Target URL
+      # @param body [String, nil] Request body
+      # @param options [Hash] Request options
+      # @return [NetHttpResponse] Response wrapper
+      def post(url, body: nil, **options)
+        uri = URI.parse(url)
+        req = Net::HTTP::Post.new(uri)
+        req.body = body if body
+        request(uri, req, options)
+      end
+      private
+      def request(uri, req, options)
+        @headers.each { |k, v| req[k] = v }
+        http = Net::HTTP.new(
+          uri.host,
+          uri.port,
+          @proxy_config[:host],
+          @proxy_config[:port],
+          @proxy_config[:username],
+          @proxy_config[:password]
+        )
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = @timeouts[:connect]
+        http.read_timeout = @timeouts[:read]
+        # Apply SSL context
+        ctx = options[:ssl_context] || @ssl_context
+        http.verify_mode = ctx.verify_mode if ctx
+        response = http.request(req)
+        NetHttpResponse.new(response)
+      end
+    end
+    # Response wrapper that mimics http.rb response interface
+    #
+    # @api private
+    class NetHttpResponse
+      # @return [Net::HTTPResponse] Raw response
+      attr_reader :raw
+      def initialize(response)
+        @raw = response
+      end
+      # @return [String] Response body
+      def body
+        @raw.body
+      end
+      # @return [Integer] HTTP status code
+      def status
+        @raw.code.to_i
+      end
+      # @return [Hash] Response headers
+      def headers
+        @raw.to_hash.transform_values(&:first)
+      end
+      # Alias for body (http.rb compatibility)
+      def to_s
+        body
+      end
+    end
+  end
+end

data/lib/zenrows/client.rb CHANGED Viewed

@@ -13,12 +13,17 @@ module Zenrows
   #
   #   client = Zenrows::Client.new
   #   http = client.http(js_render: true)
-  #   response = http.get('https://example.com', ssl_context: client.ssl_context)
+  #   response = http.get('https://example.com')
   #
   # @example With custom configuration
   #   client = Zenrows::Client.new(api_key: 'KEY', host: 'proxy.zenrows.com')
   #   http = client.http(premium_proxy: true, proxy_country: 'us')
   #
+  # @example With per-client hooks
+  #   client = Zenrows::Client.new do |c|
+  #     c.on_response { |resp, ctx| puts "#{ctx[:host]} -> #{resp.status}" }
+  #   end
+  #
   # @author Ernest Bursa
   # @since 0.1.0
   # @api public
@@ -32,17 +37,30 @@ module Zenrows
     # @return [Backends::Base] HTTP backend instance
     attr_reader :backend
+    # @return [Hooks] Hook registry for this client
+    attr_reader :hooks
     # Initialize a new client
     #
     # @param api_key [String, nil] Override API key from global config
     # @param host [String, nil] Override proxy host
     # @param port [Integer, nil] Override proxy port
     # @param backend [Symbol] Backend to use (:http_rb)
+    # @yield [config] Optional block for per-client configuration (hooks, etc.)
+    # @yieldparam config [Configuration] Client configuration for hook registration
     # @raise [ConfigurationError] if api_key is not configured
-    def initialize(api_key: nil, host: nil, port: nil, backend: nil)
+    #
+    # @example With per-client hooks
+    #   client = Zenrows::Client.new do |c|
+    #     c.on_response { |resp, ctx| puts resp.status }
+    #   end
+    def initialize(api_key: nil, host: nil, port: nil, backend: nil, &block)
       @config = build_config(api_key: api_key, host: host, port: port, backend: backend)
       @config.validate!
+      # Build hooks: start with global, allow per-client additions
+      @hooks = block ? build_hooks(&block) : Zenrows.configuration.hooks.dup
       @proxy = Proxy.new(
         api_key: @config.api_key,
         host: @config.host,
@@ -74,11 +92,11 @@ module Zenrows
     #
     # @example Basic request
     #   http = client.http(js_render: true)
-    #   response = http.get(url, ssl_context: client.ssl_context)
+    #   response = http.get(url)
     #
     # @example With premium proxy and country
     #   http = client.http(premium_proxy: true, proxy_country: 'us')
-    #   response = http.get(url, ssl_context: client.ssl_context)
+    #   response = http.get(url)
     def http(options = {})
       backend.build_client(options)
     end
@@ -86,12 +104,10 @@ module Zenrows
     # Get SSL context for proxy connections
     #
     # ZenRows proxy requires SSL verification to be disabled.
+    # This is automatically applied when using #http, but exposed
+    # for advanced use cases.
     #
     # @return [OpenSSL::SSL::SSLContext] SSL context
-    #
-    # @example
-    #   http = client.http(js_render: true)
-    #   response = http.get(url, ssl_context: client.ssl_context)
     def ssl_context
       backend.ssl_context
     end
@@ -148,12 +164,105 @@ module Zenrows
     # @return [Backends::Base] Backend instance
     # @raise [ConfigurationError] if backend is not supported
     def build_backend
-      case config.backend
+      backend_name = resolve_backend
+      case backend_name
       when :http_rb
-        Backends::HttpRb.new(proxy: proxy, config: config)
+        Backends::HttpRb.new(proxy: proxy, config: config, hooks: hooks)
+      when :net_http
+        Backends::NetHttp.new(proxy: proxy, config: config, hooks: hooks)
       else
-        raise ConfigurationError, "Unsupported backend: #{config.backend}. Use :http_rb"
+        raise ConfigurationError, "Unsupported backend: #{backend_name}. Use :http_rb or :net_http"
+      end
+    end
+    # Build hooks registry for this client
+    #
+    # Starts with global hooks, then applies per-client hooks from block.
+    #
+    # @yield [config] Block for registering per-client hooks
+    # @return [Hooks] Combined hooks registry
+    def build_hooks
+      # Start with a copy of global hooks
+      client_hooks = Zenrows.configuration.hooks.dup
+      # Create a temporary config-like object for hook registration
+      hook_config = HookConfigurator.new(client_hooks)
+      yield(hook_config)
+      client_hooks
+    end
+    # Resolve which backend to use
+    #
+    # @return [Symbol] Backend name
+    def resolve_backend
+      return config.backend if config.backend == :net_http
+      # Try http_rb first (preferred), fallback to net_http
+      if config.backend == :http_rb
+        return :http_rb if http_rb_available?
+        return :net_http
       end
+      config.backend
+    end
+    # Check if http.rb gem is available
+    #
+    # @return [Boolean]
+    def http_rb_available?
+      require "http"
+      true
+    rescue LoadError
+      false
+    end
+  end
+  # Helper class for per-client hook configuration
+  #
+  # Provides the same hook registration DSL as Configuration.
+  #
+  # @api private
+  class HookConfigurator
+    # @param hooks [Hooks] Hook registry to configure
+    def initialize(hooks)
+      @hooks = hooks
+    end
+    # Register a before_request callback
+    def before_request(callable = nil, &block)
+      @hooks.register(:before_request, callable, &block)
+      self
+    end
+    # Register an after_request callback
+    def after_request(callable = nil, &block)
+      @hooks.register(:after_request, callable, &block)
+      self
+    end
+    # Register an on_response callback
+    def on_response(callable = nil, &block)
+      @hooks.register(:on_response, callable, &block)
+      self
+    end
+    # Register an on_error callback
+    def on_error(callable = nil, &block)
+      @hooks.register(:on_error, callable, &block)
+      self
+    end
+    # Register an around_request callback
+    def around_request(callable = nil, &block)
+      @hooks.register(:around_request, callable, &block)
+      self
+    end
+    # Add a subscriber object
+    def add_subscriber(subscriber)
+      @hooks.add_subscriber(subscriber)
+      self
     end
   end
 end