crawlr 0.1.0

data/lib/crawlr/config.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module Crawlr
+  # Configuration management class for Crawlr scraping sessions.
+  #
+  # The Config class centralizes all configuration options for the Crawlr framework,
+  # providing sensible defaults while allowing extensive customization of scraping
+  # behavior, networking settings, error handling, and crawling policies.
+  #
+  # @example Basic configuration
+  #   config = Crawlr::Config.new(
+  #     timeout: 15,
+  #     max_depth: 3,
+  #     max_parallelism: 5
+  #   )
+  #
+  # @example Advanced configuration with domain filtering and retries
+  #   config = Crawlr::Config.new(
+  #     allowed_domains: ['example.com', 'api.example.com'],
+  #     max_retries: 3,
+  #     retry_delay: 2.0,
+  #     retry_backoff: 1.5,
+  #     random_delay: 1.0,
+  #     allow_cookies: true,
+  #     ignore_robots_txt: false
+  #   )
+  #
+  # @example Proxy configuration
+  #   config = Crawlr::Config.new(
+  #     proxies: ['proxy1.com:8080', 'proxy2.com:8080'],
+  #     proxy_strategy: :random,
+  #     max_parallelism: 10
+  #   )
+  #
+  # @author [Your Name]
+  # @since 0.1.0
+  class Config
+    # @return [Integer] HTTP request timeout in seconds
+    # @return [Hash<String, String>] Default HTTP headers for all requests
+    # @return [Array<String>] Glob patterns for allowed domains
+    # @return [Array<String>] Explicit list of allowed domains
+    # @return [Boolean] Whether to enable cookie handling
+    # @return [Integer] Maximum crawling depth (0 for unlimited)
+    # @return [Float] Maximum random delay between requests in seconds
+    # @return [Integer] Maximum number of concurrent requests
+    # @return [Boolean] Whether to allow revisiting previously scraped URLs
+    # @return [Integer, nil] Maximum number of retry attempts (nil to disable)
+    # @return [Float] Base delay between retry attempts in seconds
+    # @return [Float] Exponential backoff multiplier for retry delays
+    # @return [Array<Class>] List of exception classes that trigger retries
+    # @return [Integer] Maximum number of URLs to track in visit history
+    # @return [Array<String>] List of proxy server addresses
+    # @return [Symbol] Strategy for selecting proxies (:round_robin, :random)
+    # @return [Boolean] Whether to ignore robots.txt restrictions
+    attr_accessor :timeout, :headers, :domain_glob, :allowed_domains, :allow_cookies,
+                  :max_depth, :random_delay, :max_parallelism, :allow_url_revisit,
+                  :max_retries, :retry_delay, :retry_backoff, :retryable_errors,
+                  :max_visited, :proxies, :proxy_strategy, :ignore_robots_txt
+
+    # Initializes a new Config instance with the provided options
+    #
+    # @param options [Hash] Configuration options hash
+    # @option options [Integer] :timeout (10) HTTP request timeout in seconds
+    # @option options [Hash<String, String>] :default_headers Default HTTP headers
+    # @option options [Array<String>] :allowed_domains ([]) Explicit list of allowed domains
+    # @option options [Array<String>] :domain_glob ([]) Glob patterns for domain filtering
+    # @option options [Boolean] :allow_cookies (false) Enable cookie handling
+    # @option options [Integer] :max_depth (0) Maximum crawling depth (0 = unlimited)
+    # @option options [Float] :random_delay (0) Maximum random delay between requests
+    # @option options [Integer] :max_parallelism (1) Maximum concurrent requests
+    # @option options [Boolean] :allow_url_revisit (false) Allow revisiting URLs
+    # @option options [Integer] :max_retries (0) Maximum retry attempts (0 = disabled)
+    # @option options [Float] :retry_delay (1.0) Base retry delay in seconds
+    # @option options [Float] :retry_backoff (2.0) Exponential backoff multiplier
+    # @option options [Array<Class>] :retryable_errors Custom list of retryable exceptions
+    # @option options [Integer] :max_visited (10000) Maximum URLs to track in history
+    # @option options [Array<String>] :proxies ([]) List of proxy servers
+    # @option options [Symbol] :proxy_strategy (:round_robin) Proxy selection strategy
+    # @option options [Boolean] :ignore_robots_txt (false) Ignore robots.txt restrictions
+    #
+    # @raise [StandardError] When both :allowed_domains and :domain_glob are specified
+    #
+    # @example Minimal configuration
+    #   config = Crawlr::Config.new
+    #
+    # @example Timeout and parallelism configuration
+    #   config = Crawlr::Config.new(
+    #     timeout: 30,
+    #     max_parallelism: 8
+    #   )
+    #
+    # @example Domain filtering with explicit domains
+    #   config = Crawlr::Config.new(
+    #     allowed_domains: ['site1.com', 'api.site1.com']
+    #   )
+    #
+    # @example Domain filtering with glob patterns
+    #   config = Crawlr::Config.new(
+    #     domain_glob: ['*.example.com', '*.api.example.com']
+    #   )
+    #
+    # @example Retry configuration with custom errors
+    #   config = Crawlr::Config.new(
+    #     max_retries: 5,
+    #     retry_delay: 0.5,
+    #     retry_backoff: 1.5,
+    #     retryable_errors: [Timeout::Error, Net::ReadTimeout]
+    #   )
+    def initialize(options = {})
+      initialize_domain_settings(options)
+      initialize_parallelism_settings(options)
+      initialize_throttle_settings(options)
+      initialize_http_settings(options)
+      initialize_retry_settings(options)
+      initialize_visit_settings(options)
+      initialize_proxy_settings(options)
+      initialize_robots_settings(options)
+
+      validate
+    end
+
+    # Converts the configuration to a hash representation
+    #
+    # This method is useful for serialization, debugging, or creating
+    # new Config instances with the same settings.
+    #
+    # @return [Hash<Symbol, Object>] Hash containing all configuration values
+    #
+    # @example
+    #   config = Crawlr::Config.new(timeout: 15, max_depth: 3)
+    #   hash = config.to_h
+    #   new_config = Crawlr::Config.new(hash)
+    #
+    # @example Inspect configuration
+    #   puts config.to_h.inspect
+    def to_h
+      attrs = %i[
+        timeout headers allowed_domains domain_glob allow_cookies max_depth
+        random_delay max_parallelism allow_url_revisit max_retries retry_delay
+        retry_backoff retryable_errors max_visited proxies proxy_strategy
+        ignore_robots_txt
+      ]
+
+      attrs.each_with_object({}) { |name, hash| hash[name] = instance_variable_get("@#{name}") }
+    end
+
+    private
+
+    def initialize_domain_settings(options)
+      @allowed_domains = Array(options[:allowed_domains])
+      @domain_glob     = Array(options[:domain_glob])
+    end
+
+    def initialize_parallelism_settings(options)
+      @max_parallelism = options.fetch(:max_parallelism, 1)
+    end
+
+    def initialize_throttle_settings(options)
+      @random_delay = options.fetch(:random_delay, 0)
+    end
+
+    def initialize_http_settings(options)
+      @timeout      = options.fetch(:timeout, 10)
+      @headers      = options[:default_headers] || default_headers
+      @allow_cookies = options.fetch(:allow_cookies, false)
+      @max_depth = options.fetch(:max_depth, 0)
+    end
+
+    def initialize_retry_settings(options)
+      @max_retries       = options[:max_retries]&.positive? ? options[:max_retries] : 0
+      @retry_delay       = options.fetch(:retry_delay, 1.0)
+      @retry_backoff     = options.fetch(:retry_backoff, 2.0)
+      @retryable_errors  = options[:retryable_errors] || default_retryable_errors
+    end
+
+    def initialize_visit_settings(options)
+      @allow_url_revisit = options.fetch(:allow_url_revisit, false)
+      @max_visited       = options.fetch(:max_visited, 10_000)
+    end
+
+    def initialize_proxy_settings(options)
+      @proxies        = Array(options[:proxies])
+      @proxy_strategy = options.fetch(:proxy_strategy, :round_robin)
+    end
+
+    def initialize_robots_settings(options)
+      @ignore_robots_txt = options.fetch(:ignore_robots_txt, false)
+    end
+
+    # Returns the default HTTP headers for requests
+    #
+    # @return [Hash<String, String>] Default headers with User-Agent
+    # @api private
+    def default_headers
+      {
+        "User-Agent" => "Crawlr/#{Crawlr::VERSION}"
+      }
+    end
+
+    # Returns the default list of exceptions that should trigger retries
+    #
+    # These exceptions typically represent temporary network issues
+    # that may resolve on subsequent attempts.
+    #
+    # @return [Array<Class>] Array of exception classes for retry logic
+    # @api private
+    def default_retryable_errors
+      [
+        Async::TimeoutError,
+        Errno::ECONNREFUSED,
+        Errno::ECONNRESET,
+        Errno::EHOSTUNREACH,
+        Errno::ENETUNREACH,
+        SocketError
+      ]
+    end
+
+    # Validates the configuration for conflicting options
+    #
+    # Ensures that mutually exclusive configuration options are not
+    # specified simultaneously, which would create ambiguous behavior.
+    #
+    # @return [void]
+    # @raise [StandardError] When both allowed_domains and domain_glob are specified
+    # @api private
+    def validate
+      return unless !@allowed_domains.empty? && !@domain_glob.empty?
+
+      raise "Cannot specify both allowed_domains and domain_glob"
+    end
+  end
+end

data/lib/crawlr/context.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Crawlr
+  # The Context class holds metadata and shared data
+  # during a scraping session, such as URLs and crawl depth.
+  #
+  # It acts like a small key-value store (`@data`) and provides
+  # helper methods to manage depth and resolve relative URLs.
+  #
+  # @example Creating a new context
+  #   ctx = Crawlr::Context.new(base_url: "https://example.com")
+  #   ctx[:title] = "Home"
+  #   ctx.increment_depth
+  #   ctx.to_h
+  #   # => { base_url: "https://example.com", page_url: nil, current_depth: 1, title: "Home" }
+  #
+  class Context
+    # @return [String, nil] The base URL used for resolving relative links
+    # @return [String, nil] The current page URL
+    # @return [Integer] The current depth in the crawl hierarchy
+    attr_accessor :base_url, :page_url, :current_depth
+
+    # Create a new scraping context.
+    #
+    # @param [String, nil] base_url The root URL of the crawl
+    # @param [String, nil] page_url The current page URL
+    # @param [Integer] current_depth The crawl depth (default: 0)
+    def initialize(base_url: nil, page_url: nil, current_depth: 0)
+      @base_url = base_url
+      @page_url = page_url
+      @current_depth = current_depth
+      @data = {}
+    end
+
+    # Retrieve a stored value by key.
+    #
+    # @param [Symbol, String] key The key to fetch
+    # @return [Object, nil] The stored value, or nil if not found
+    def [](key)
+      @data[key]
+    end
+
+    # Assign a value to a key.
+    #
+    # @param [Symbol, String] key The key to set
+    # @param [Object] value The value to store
+    # @return [Object] The stored value
+    def []=(key, value)
+      @data[key] = value
+    end
+
+    # Convert the context to a Hash.
+    #
+    # Includes base_url, page_url, current_depth, and all stored data.
+    #
+    # @return [Hash] The full context data as a Hash
+    def to_h
+      {
+        base_url: @base_url,
+        page_url: @page_url,
+        current_depth: @current_depth
+      }.merge(@data)
+    end
+
+    # Increment the crawl depth by 1.
+    #
+    # @return [Integer] The updated depth value
+    def increment_depth
+      @current_depth += 1
+    end
+
+    # Resolve a relative URL using the base_url.
+    #
+    # @param [String] url The relative or absolute URL
+    # @return [String] The resolved absolute URL
+    def resolve_url(url)
+      URI.join(@base_url, url).to_s
+    end
+  end
+end

data/lib/crawlr/domains.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Crawlr
+  # Domain filtering and validation class for controlling scraping scope.
+  #
+  # The Domains class manages which domains are allowed to be scraped by
+  # implementing both explicit domain allowlists and glob pattern matching.
+  # It provides flexible domain filtering to restrict scraping to specific
+  # sites or domain patterns while normalizing domain names for consistent
+  # comparison.
+  #
+  # @example Allow specific domains
+  #   config = Crawlr::Config.new(
+  #     allowed_domains: ['example.com', 'api.example.com', 'subdomain.site.org']
+  #   )
+  #   domains = Crawlr::Domains.new(config)
+  #
+  #   domains.allowed?('https://example.com/page')      #=> true
+  #   domains.allowed?('https://www.example.com/page')  #=> true (www. stripped)
+  #   domains.allowed?('https://forbidden.com/page')    #=> false
+  #
+  # @example Use glob patterns for flexible matching
+  #   config = Crawlr::Config.new(
+  #     domain_glob: ['*.example.com', '*.api.*.com', 'site?.org']
+  #   )
+  #   domains = Crawlr::Domains.new(config)
+  #
+  #   domains.allowed?('https://sub.example.com/path')     #=> true
+  #   domains.allowed?('https://api.service.com/data')     #=> true
+  #   domains.allowed?('https://site1.org/content')        #=> true
+  #
+  # @example No restrictions (allow all domains)
+  #   config = Crawlr::Config.new  # No domain restrictions
+  #   domains = Crawlr::Domains.new(config)
+  #
+  #   domains.allowed?('https://any-site.com')  #=> true
+  #
+  # @author [Your Name]
+  # @since 0.1.0
+  class Domains
+    # Initializes a new Domains instance with the given configuration
+    #
+    # @param config [Crawlr::Config] Configuration object containing domain restrictions
+    #
+    # @example
+    #   config = Crawlr::Config.new(allowed_domains: ['site.com'])
+    #   domains = Crawlr::Domains.new(config)
+    def initialize(config)
+      @config = config
+      @allowed_domains = extract_allowed_domains(@config.allowed_domains)
+      @domain_glob = @config.domain_glob
+    end
+
+    # Checks if a URL is allowed based on configured domain restrictions
+    #
+    # The method performs the following checks in order:
+    # 1. If no restrictions are configured, allows all URLs
+    # 2. If glob patterns are configured, tests URL against each pattern
+    # 3. If explicit domains are configured, checks normalized domain name
+    # 4. Logs rejection for debugging purposes
+    #
+    # @param url [String] The URL to check for domain allowance
+    # @return [Boolean] true if the URL's domain is allowed, false otherwise
+    #
+    # @example With explicit domain allowlist
+    #   domains.allowed?('https://example.com/page')        #=> true (if allowed)
+    #   domains.allowed?('https://www.example.com/page')    #=> true (www. stripped)
+    #   domains.allowed?('https://subdomain.example.com')   #=> false (unless explicitly allowed)
+    #
+    # @example With glob patterns
+    #   # config.domain_glob = ['*.example.com']
+    #   domains.allowed?('https://api.example.com')         #=> true
+    #   domains.allowed?('https://cdn.example.com/asset')   #=> true
+    #   domains.allowed?('https://other.com')               #=> false
+    #
+    # @example No restrictions
+    #   # config.allowed_domains = [], config.domain_glob = []
+    #   domains.allowed?('https://any-domain.com')          #=> true
+    def allowed?(url)
+      return true if @allowed_domains.empty? && @domain_glob.empty?
+
+      unless @domain_glob.empty?
+        @domain_glob.each do |glob|
+          return true if File.fnmatch?(glob, url)
+        end
+      end
+
+      uri = URI(url)
+      base_name = uri.host.sub("www.", "")
+      allowed = @allowed_domains.include?(base_name)
+
+      Crawlr.logger.info("URL not allowed: #{url}") unless allowed
+      allowed
+    end
+
+    # Returns statistics about the configured domain restrictions
+    #
+    # Provides metrics about the number of explicitly allowed domains
+    # and glob patterns configured for monitoring and debugging purposes.
+    #
+    # @return [Hash<Symbol, Integer>] Statistics hash containing domain counts
+    # @option return [Integer] :allowed_domains Number of explicitly allowed domains
+    # @option return [Integer] :domain_glob Number of configured glob patterns
+    #
+    # @example
+    #   stats = domains.domain_stats
+    #   puts "Allowing #{stats[:allowed_domains]} explicit domains"
+    #   puts "Using #{stats[:domain_glob]} glob patterns"
+    def domain_stats
+      {
+        allowed_domains: @allowed_domains.size,
+        domain_glob: @domain_glob.size
+      }
+    end
+
+    private
+
+    # Extracts and normalizes domain names from the configuration
+    #
+    # Processes the list of allowed domains by:
+    # 1. Handling nil/empty input gracefully
+    # 2. Normalizing each domain using base_domain method
+    # 3. Removing duplicates from the final list
+    #
+    # @param domains [Array<String>, nil] List of domain strings to process
+    # @return [Array<String>] Normalized, unique list of base domain names
+    # @api private
+    #
+    # @example
+    #   extract_allowed_domains(['https://www.example.com', 'api.example.com'])
+    #   #=> ['example.com', 'api.example.com']
+    def extract_allowed_domains(domains)
+      return [] if domains.nil? || domains.empty?
+
+      domains.map { |domain| base_domain(domain) }.uniq
+    end
+
+    # Normalizes a domain string to its base form for consistent comparison
+    #
+    # Performs the following normalization:
+    # 1. Parses as URI if it looks like a full URL
+    # 2. Ensures path is set to "/" if empty (for valid URI)
+    # 3. Extracts hostname and removes "www." prefix
+    # 4. Falls back to original string if URI parsing fails
+    #
+    # @param domain [String] Domain string or URL to normalize
+    # @return [String] Normalized base domain name without www prefix
+    # @api private
+    #
+    # @example URL normalization
+    #   base_domain('https://www.example.com/path') #=> 'example.com'
+    #   base_domain('http://api.site.org')          #=> 'api.site.org'
+    #
+    # @example Domain name normalization
+    #   base_domain('www.example.com')              #=> 'example.com'
+    #   base_domain('subdomain.example.com')        #=> 'subdomain.example.com'
+    #
+    # @example Fallback behavior
+    #   base_domain('not-a-valid-uri')              #=> 'not-a-valid-uri'
+    def base_domain(domain)
+      uri = URI(domain)
+      uri.path = "/" if uri.path.empty?
+      uri.host ? uri.host.sub("www.", "") : domain
+    end
+  end
+end

data/lib/crawlr/hooks.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Crawlr
+  # Event hook management system for scraping lifecycle customization.
+  #
+  # The Hooks class provides a flexible event-driven system that allows users
+  # to register custom behavior at specific points during the scraping process.
+  # It supports multiple hooks per event and validates event names to ensure
+  # consistency across the framework.
+  #
+  # @example Basic hook registration
+  #   hooks = Crawlr::Hooks.new
+  #
+  #   hooks.register(:before_visit) do |url, headers|
+  #     puts "About to visit: #{url}"
+  #     headers['X-Custom'] = 'value'
+  #   end
+  #
+  # @example Multiple hooks for the same event
+  #   hooks.register(:after_visit) do |url, response|
+  #     log_response_time(url, response)
+  #   end
+  #
+  #   hooks.register(:after_visit) do |url, response|
+  #     update_statistics(response.status)
+  #   end
+  #
+  # @example Error handling hooks
+  #   hooks.register(:on_error) do |url, error|
+  #     error_logger.warn("Failed to scrape #{url}: #{error.message}")
+  #     notify_monitoring_system(url, error)
+  #   end
+  #
+  # @author [Your Name]
+  # @since 0.1.0
+  class Hooks
+    # Supported lifecycle events for hook registration
+    #
+    # @return [Array<Symbol>] Array of valid event names
+    # - `:before_visit` - Triggered before making HTTP request
+    # - `:after_visit` - Triggered after receiving HTTP response
+    # - `:on_error` - Triggered when an error occurs during scraping
+    ALLOWED_EVENTS = %i[before_visit after_visit on_error].freeze
+
+    # Initializes a new Hooks instance
+    #
+    # Creates an empty hook registry with auto-vivifying arrays for each event type.
+    #
+    # @example
+    #   hooks = Crawlr::Hooks.new
+    def initialize
+      @hooks = Hash.new { |h, k| h[k] = [] }
+    end
+
+    # Registers a hook for a specific scraping lifecycle event
+    #
+    # Hooks are executed in the order they were registered. Multiple hooks
+    # can be registered for the same event, and all will be executed when
+    # the event is triggered.
+    #
+    # @param event [Symbol] The lifecycle event to hook into
+    # @param block [Proc] The block to execute when the event occurs
+    # @yieldparam args [Array] Event-specific arguments passed to the hook
+    # @return [void]
+    # @raise [ArgumentError] When the event is not in ALLOWED_EVENTS
+    # @raise [ArgumentError] When no block is provided
+    #
+    # @example Before visit hook for request modification
+    #   register(:before_visit) do |url, headers|
+    #     headers['User-Agent'] = 'Custom Bot 1.0'
+    #     headers['Authorization'] = get_auth_token(url)
+    #   end
+    #
+    # @example After visit hook for response processing
+    #   register(:after_visit) do |url, response|
+    #     response_time = response.headers['X-Response-Time']
+    #     metrics.record_response_time(url, response_time)
+    #   end
+    #
+    # @example Error handling hook
+    #   register(:on_error) do |url, error|
+    #     if error.is_a?(Timeout::Error)
+    #       retry_queue.add(url, delay: 30)
+    #     end
+    #   end
+    def register(event, &block)
+      raise ArgumentError, "Invalid event #{event}" unless ALLOWED_EVENTS.include?(event)
+      raise ArgumentError, "Block required" unless block
+
+      @hooks[event] << block
+    end
+
+    # Triggers all registered hooks for a specific event
+    #
+    # Executes hooks in the order they were registered. If any hook raises
+    # an exception, it will be propagated and may prevent subsequent hooks
+    # from executing.
+    #
+    # @param event [Symbol] The event to trigger
+    # @param args [Array] Variable arguments to pass to the hook blocks
+    # @return [void]
+    # @raise [ArgumentError] When the event is not in ALLOWED_EVENTS
+    #
+    # @example Trigger before_visit hooks
+    #   trigger(:before_visit, 'https://example.com', headers_hash)
+    #
+    # @example Trigger after_visit hooks
+    #   trigger(:after_visit, 'https://example.com', response_object)
+    #
+    # @example Trigger error hooks
+    #   trigger(:on_error, 'https://example.com', exception_object)
+    def trigger(event, *args)
+      raise ArgumentError, "Invalid event #{event}" unless ALLOWED_EVENTS.include?(event)
+
+      @hooks[event].each { |blk| blk.call(*args) }
+    end
+
+    # Returns statistics about registered hooks
+    #
+    # Provides metrics about hook registration for monitoring, debugging,
+    # and ensuring expected hooks are properly configured.
+    #
+    # @return [Hash<Symbol, Object>] Statistics hash containing hook metrics
+    # @option return [Integer] :total_hooks Total number of registered hooks across all events
+    # @option return [Hash<Symbol, Integer>] :per_event Number of hooks per event type
+    #
+    # @example
+    #   stats = hooks.stats
+    #   puts "Total hooks: #{stats[:total_hooks]}"
+    #   puts "Before visit hooks: #{stats[:per_event][:before_visit]}"
+    #   puts "Error hooks: #{stats[:per_event][:on_error]}"
+    def stats
+      grouped = @hooks.transform_values(&:size)
+      { total_hooks: @hooks.values.flatten.size, per_event: grouped }
+    end
+
+    # Clears registered hooks for all events or a specific event
+    #
+    # Useful for testing, resetting hook configuration, or dynamically
+    # changing hook behavior during scraping sessions.
+    #
+    # @param event [Symbol, nil] Specific event to clear, or nil to clear all
+    # @return [void]
+    #
+    # @example Clear all hooks
+    #   hooks.clear
+    #
+    # @example Clear hooks for specific event
+    #   hooks.clear(:before_visit)
+    #
+    # @example Clear error hooks only
+    #   hooks.clear(:on_error)
+    def clear(event = nil)
+      if event
+        @hooks[event].clear
+      else
+        @hooks.clear
+      end
+    end
+  end
+end