wayfarer 0.4.6 → 0.4.8

data/lib/wayfarer/networking/http.rb

@@ -9,7 +9,7 @@ module Wayfarer
 
       def create
         Net::HTTP::Persistent.new(name: CONNECTION_NAME).tap do |conn|
-          Wayfarer.config.network.http_headers.each do |key, val|
+          Wayfarer.config[:network][:http_headers].each do |key, val|
             conn.override_headers[key] = val
           end
         end

data/lib/wayfarer/networking/pool.rb

@@ -5,25 +5,35 @@ module Wayfarer
     class Pool
       include Singleton
 
-      cattr_accessor :registry, default: { http: HTTP,
-                                           ferrum: Ferrum,
-                                           selenium: Selenium,
-                                           capybara: Capybara }
-
-      def pool
-        @pool ||= ConnectionPool.new(size: Wayfarer.config.network.pool_size,
-                                     timeout: Wayfarer.config.network.pool_timeout,
-                                     &method(:context))
+      class_attribute :finalizer,
+                      default: ->(pool) { pool.shutdown(&:renew) }.freeze,
+                      instance_accessor: false,
+                      instance_predicate: false
+
+      class_attribute :registry,
+                      default: { http: HTTP,
+                                 ferrum: Ferrum,
+                                 selenium: Selenium,
+                                 capybara: Capybara },
+                      instance_accessor: false,
+                      instance_predicate: false
+
+      attr_reader :pool
+
+      def initialize
+        @pool = ConnectionPool.new(**Wayfarer.config.dig(:network, :pool), &method(:context))
+
+        at_exit { free }
       end
 
-      def with(&block)
-        pool.with(&block)
+      def with(&)
+        @pool.with(&)
       rescue ConnectionPool::TimeoutError => e
         raise Wayfarer::UserAgentTimeoutError, e
       end
 
       def free
-        pool.shutdown(&:renew)
+        self.class.finalizer.call(@pool)
       end
 
     private
@@ -33,7 +43,7 @@ module Wayfarer
       end
 
       def strategy
-        self.class.registry[Wayfarer.config.network.agent].new
+        self.class.registry[Wayfarer.config.dig(:network, :agent)].new
       end
     end
   end

data/lib/wayfarer/networking/selenium.rb

@@ -9,7 +9,7 @@ module Wayfarer
       MOCK_RESPONSE_HEADERS = {}.freeze
 
       def create
-        ::Selenium::WebDriver.for(Wayfarer.config.selenium.driver, **options)
+        ::Selenium::WebDriver.for(driver, options)
       end
 
       def destroy(instance)
@@ -29,15 +29,23 @@ module Wayfarer
 
     private
 
-      def options
-        Wayfarer.config.selenium.options.merge(http_client: http_client)
+      def driver
+        Wayfarer.config.dig(:selenium, :driver)
       end
 
-      def http_client
-        ::Selenium::WebDriver::Remote::Http::Default.new.tap do |client|
-          client.read_timeout = Wayfarer.config.selenium.client_timeout
-        end
+      def options
+        Wayfarer.config.dig(:selenium, :options)
       end
+
+      # def options
+      #   Wayfarer.config[:selenium][:options].merge(http_client: http_client)
+      # end
+
+      # def http_client
+      #   ::Selenium::WebDriver::Remote::Http::Default.new.tap do |client|
+      #     client.read_timeout = Wayfarer.config[:selenium][:client_timeout]
+      #   end
+      # end
     end
   end
 end

data/lib/wayfarer/networking/strategy.rb

@@ -13,13 +13,13 @@ module Wayfarer
       end
 
       def navigate(_instance, _url)
-        raise NoMethodError
+        raise NotImplementedError
       end
 
       def live(_instance); end
 
       def create
-        raise NoMethodError
+        raise NotImplementedError
       end
 
       def destroy(_instance); end

data/lib/wayfarer/page.rb

@@ -1,12 +1,22 @@
 # frozen_string_literal: true
 
 module Wayfarer
+  # @!attribute [r] url
+  #   @return [String] the URL that was fetched
+  # @!attribute [r] status_code
+  #   @return [Fixnum] HTTP status code
+  # @!attribute [r] body
+  #   @return [String] the body of the response
+  # @!attribute [r] headers
+  #   @return [Hash] the headers of the response
+  #   @note HTTP header keys are downcased, for example: `content-type`.
   class Page
     attr_reader :url,
                 :status_code,
                 :body,
                 :headers
 
+    # @!visibility private
     def initialize(url:, status_code:, body:, headers:)
       @url = url
       @status_code = status_code
@@ -14,24 +24,34 @@ module Wayfarer
       @headers = headers.transform_keys(&:downcase)
     end
 
-    def doc
-      return @doc if @doc
-
-      # If no Content-Type field is present, assume HTML/XML
-      return @doc = Wayfarer::Parsing::XML.parse_html(body) unless headers["content-type"]
-
-      content_type = headers["content-type"]
-      sub_type = MIME::Types[content_type].first.sub_type
+    # Returns the MIME type of the response.
+    # @return [MIME::Type]
+    # @see https://www.rubydoc.info/gems/mime-types/MIME/Type
+    def mime_type
+      @mime_type ||= MIME::Types[content_type]&.first
+    end
 
-      @doc = case sub_type
-             when "json" then Wayfarer::Parsing::JSON.parse(body)
-             when "xml" then Wayfarer::Parsing::XML.parse_xml(body)
-             else Wayfarer::Parsing::XML.parse_html(body)
-             end
+    # Returns a parsed representation of the HTTP response or the browser DOM,
+    # depending on the Content-Type.
+    # @return [Nokogiri::HTML::Document] when Content-Type is `text/html`
+    # @see https://www.rubydoc.info/github/sparklemotion/nokogiri/Nokogiri/HTML/Document Nokogiri::HTML::Document
+    # @return [Nokogiri::XML::Document] when Content-Type is `text/xml`
+    # @see https://www.rubydoc.info/github/sparklemotion/nokogiri/Nokogiri/XML/Document Nokogiri::XML::Document
+    # @return [Hash] when Content-Type is `application/json`
+    # @note You can register custom parsers with {Wayfarer::Parsing.registry}.
+    def doc
+      @doc ||= Wayfarer::Parsing.parse(body, mime_type&.content_type || content_type)
     end
 
+    # Returns a `MetaInspector::Document`.
+    # @return [MetaInspector::Document]
+    # @see https://www.rubydoc.info/gems/metainspector/MetaInspector/Document
     def meta
-      @meta ||= MetaInspector.new(url, document: body)
+      @meta ||= MetaInspector.new(url, document: body, headers: headers, normalize_url: false)
+    end
+
+    def content_type
+      @content_type ||= headers["content-type"]
     end
   end
 end

data/lib/wayfarer/parsing/xml.rb

@@ -5,12 +5,12 @@ module Wayfarer
     module XML
     module_function
 
-      def parse_xml(xml)
-        Nokogiri::XML(xml)
-      end
-
-      def parse_html(html)
-        Nokogiri::HTML(html)
+      def parse(xml, variant)
+        case variant
+        when :xml then Nokogiri::XML(xml)
+        when :html then Nokogiri::HTML(xml)
+        else raise ArgumentError, "Unknown type: #{variant}"
+        end
       end
     end
   end

data/lib/wayfarer/parsing.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  # @!scope class
+  # @!attribute [r] registry
+  #   @return [Hash] Mapping of Content-Type to parser.
+  module Parsing
+    # @!visibility private
+    FALLBACK_CONTENT_TYPE = "application/octet-stream"
+
+  module_function
+
+    # @!visibility private
+    def parse(body, content_type = FALLBACK_CONTENT_TYPE)
+      parser, args = Wayfarer.config.dig(:parsing, :registry, content_type)
+      return unless parser
+
+      parser.parse(body, *args)
+    end
+  end
+end

data/lib/wayfarer/redis/barrier.rb

@@ -2,35 +2,40 @@
 
 module Wayfarer
   module Redis
-    Barrier = Struct.new(:batch) do
-      include Connection
+    # A Barrier prevents processing the same key more than once.
+    # It marks keys in a Redis hash as they are encountered.
+    # Once a key is added, it cannot get removed.
+    class Barrier
+      include Resettable
 
-      def redis_key
-        "wayfarer-barrier-#{batch}"
-      end
+      # @return [Hash] the task configuration, including redis_pool and batch name
+      attr_reader :task
 
-      def reset!
-        redis { |conn| conn.del(redis_key) }
-      end
+      VALUE = ""
 
-      def seen?(url)
-        !redis { |conn| conn.sadd(redis_key, url) }
+      # @param task [Wayfarer::Task] task context
+      def initialize(task)
+        @task = task
+        @redis_pool = task[:redis_pool]
       end
 
-      def peek(urls)
-        major, minor, = Version.determine
-
-        # SMISMEMBER is only supported on Redis >= 6.2.0
-        if major > 6 || (major == 6 && minor >= 2)
-          redis { |conn| conn.smismember(redis_key, urls) }.map { |val| val == 1 }
-        else
-          urls.map { |url| redis { |conn| conn.sismember(redis_key, url) } }
-        end
+      # @return [String] the Redis key for this barrier
+      def redis_key
+        "wayfarer-barrier-#{task.batch}"
       end
 
-      def unsee(url)
-        redis { |conn| conn.srem(redis_key, url) }
+      # Checks if a key has already been passed through the barrier.
+      #
+      # @param key [String] the key to check
+      # @return [Boolean] true if the key has already been seen, false otherwise
+      def check!(key)
+        !redis_pool.with { |conn| conn.hsetnx(redis_key, key, VALUE) }
       end
+
+    private
+
+      # @return [ConnectionPool] the Redis connection pool
+      attr_reader :redis_pool
     end
   end
 end

data/lib/wayfarer/redis/counter.rb

@@ -2,28 +2,37 @@
 
 module Wayfarer
   module Redis
-    Counter = Struct.new(:batch) do
-      include Connection
+    class Counter
+      include Resettable
 
-      def redis_key
-        "wayfarer-counter-#{batch}"
+      attr_reader :task
+
+      def initialize(task)
+        @task = task
+        @redis_pool = task[:redis_pool]
       end
 
-      def reset!
-        redis { |conn| conn.del(redis_key) }
+      def redis_key
+        "wayfarer-counter-#{@task.batch}"
       end
 
       def value
-        redis { |conn| conn.get(redis_key) }.to_i
+        redis_pool.with { |conn| conn.get(redis_key) }.to_i
       end
 
       def increment
-        redis { |conn| conn.incr(redis_key) }
+        redis_pool.with { |conn| conn.incr(redis_key) }
       end
 
       def decrement
-        redis { |conn| conn.decr(redis_key) }
+        redis_pool.with { |conn| conn.decr(redis_key) }.tap do |val|
+          @callback&.call if val == 0
+        end
       end
+
+    private
+
+      attr_reader :redis_pool
     end
   end
 end

data/lib/wayfarer/redis/pool.rb

@@ -10,7 +10,7 @@ module Wayfarer
 
       def initialize
         @pool = ConnectionPool.new do
-          Wayfarer.config.redis.factory.call(Wayfarer.config.redis)
+          Wayfarer.config[:redis][:factory].call(Wayfarer.config[:redis])
         end
       end
 

data/lib/wayfarer/redis/resettable.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Redis
+    module Resettable
+      def reset!
+        redis_pool.with { |conn| conn.del(redis_key) }
+      end
+
+      def redis_pool
+        raise NotImplementedError
+      end
+
+      def redis_key
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/dsl.rb

@@ -2,56 +2,192 @@
 
 module Wayfarer
   module Routing
+    # Routing DSL that declares a tree of {Route}s. Routing trees decide
+    # whether a URL gets processed. Each route has a matcher which is a
+    # predicate. Routes are searched depth-first by a {PathFinder}.
+    #
+    # When you call a DSL method on a route, you declare a child route on the
+    # route. You can pass a keyword list and a block when you call DSL methods.
+    # Keyword lists result in route chains. For example, if you declare:
+    #
+    # ```ruby
+    # route.host "example.com", path: ":foo", query: { page: 1 }
+    # ```
+    #
+    # The last route with the query matcher is returned, and the hierarchy is
+    # `host > path > query`.
+    #
+    # To append multiple child rules, pass a block within which you can call
+    # DSL methods to declare child routes:
+    #
+    # ```ruby
+    # route.host "example.com" do
+    #   path ":foo"
+    #   query page: 1
+    # end
+    # ```
+    #
+    # The query matcher is returned again, but the hierarchy is
+    # `host > path`, `host > query`.
+    #
+    # @see Route
+    # @see PathFinder
     module DSL
-      def url(url, options = {}, &block)
-        add_child_route(Matchers::URL.new(url), path_offset, options, &block)
+      # Match URLs exactly.
+      #
+      # A trailing slash in +url+ is ignored so
+      # "https://example.com" and "https://example.com/" are equivalent.
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param url [String]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Match URL exactly
+      #   route.url("https://www.iana.org/help/example-domains").to(:index)
+      def url(url, **, &)
+        child_route(Matchers::URL.new(url), **, &)
       end
 
-      def host(host, options = {}, &block)
-        add_child_route(Matchers::Host.new(host), path_offset, options, &block)
+      # Match hostnames excluding the port number.
+      #
+      # * `String` is compared literally
+      # * `Regexp` is matched against the host
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param host [String, Regexp]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Literal host
+      #   route.host("example.com").to(:home)
+      # @example Regular expression
+      #   route.host(/example\.com/) do
+      #     path "users/:id", to: :user
+      #   end
+      def host(host, **, &)
+        child_route(Matchers::Host.new(host), **, &)
       end
 
-      def path(path, options = {}, &block)
-        offset = File.join(path_offset, path)
-        add_child_route(nil, offset, options, &block).tap do |route|
-          route.matcher = Matchers::Path.new(offset, route)
-        end
-      end
+      # Match and consume path fragments. You can use Sinatra-style pattern
+      # matching to extract data from segments.
+      #
+      # @see https://github.com/sinatra/mustermann/blob/main/mustermann/README.md#-sinatra-pattern
+      #
+      # This matcher doesn't collect `params`.
+      # A leading slash is enforced on the path string.
+      #
+      # @example Capture segment
+      #   route.path(":segment").to(:show)
+      # @example Nested paths
+      #   route.path("/").path("foo").path(":id").to(:detail)
+      # @param path [String]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      def path(path, **, &)
+        path = File.join(File::SEPARATOR, path)
 
-      def query(fields, options = {}, &block)
-        add_child_route(Matchers::Query.new(fields), path_offset, options, &block)
+        child_route(Matchers::Path.new(path), **, &)
       end
 
-      def scheme(scheme, options = {}, &block)
-        add_child_route(Matchers::Scheme.new(scheme), path_offset, options, &block)
+      # Match query parameters.
+      #
+      # Each key/value pair must be present *at least once*; if multiple values
+      # occur the last one wins (like Rack).
+      #
+      # Supported value types:
+      #
+      # * `String` - exact match
+      # * `Regexp` - regular expression match
+      # * `Integer` - exact numeric match
+      # * `Range` - inclusive numeric range
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param fields [Hash{Symbol,String => String,Regexp,Integer,Range}]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Simple parameter
+      #   route.query(foo: "bar").to(:index)
+      # @example Page range
+      #   route.query(page: 5..12).to(:index)
+      def query(fields, **, &)
+        child_route(Matchers::Query.new(fields), **, &)
       end
 
-      def suffix(suffix, options = {}, &block)
-        add_child_route(Matchers::Suffix.new(suffix), path_offset, options, &block)
+      # Match URL schemes (protocols).
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param scheme [String, Symbol]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example HTTPS vs HTTP
+      #   route.scheme(:https).to(:tls)
+      #   route.scheme(:http).to(:plain)
+      def scheme(scheme, **, &)
+        child_route(Matchers::Scheme.new(scheme), **, &)
       end
 
-      def to(action, options = {}, &block)
-        add_child_route(Matchers::Custom.new { true }, path_offset, TargetRoute, options, &block).tap do |route|
-          route.action = action
-        end
+      # Declares the action for the current route branch. An action is a symbol
+      # for an instance method, or a {Wayfarer::Handler}.
+      #
+      # In case of conflicting actions for a matching route path, the last
+      # matched action takes precedence.
+      #
+      # @param action [Symbol, Wayfarer::Handler] method or {Handler} to call.
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::TargetRoute]
+      #
+      # @example Last action wins
+      #   route.to(:alpha).to(:beta) # => routes to :beta
+      def to(action, **, &)
+        child_route(nil, action: action, klass: TargetRoute, **, &)
       end
 
-      def custom(delegate, options = {}, &block)
-        add_child_route(Matchers::Custom.new(delegate), path_offset, options, &block)
+      # Match URLs dynamically by declaring a route from a block during route
+      # evaluation. Custom matchers are passed a transient root route which will
+      # be followed. Custom matchers match when their dynamically declared
+      # subtree matches the URL.
+      #
+      # @param options [Hash]
+      # @yield [root, uri, task]
+      # @yieldparam  [Wayfarer::Routing::RootRoute] root route to populate
+      # @yieldparam uri [Addressable::URI] parsed task URL
+      # @yieldparam task [Wayfarer::Task] current task
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Batch routing
+      #   route.custom do |root, _uri, task|
+      #     database_record = Crawl.find_by(batch: task.batch)
+      #     root.host(database_record.hostname_to_crawl).to(:index)
+      #   end
+      def custom(**, &block)
+        child_route(Matchers::Custom.new(block), klass: SubRoute, **)
       end
 
     private
 
-      # rubocop:disable Style/OptionalArguments
-      def add_child_route(matcher, path_offset, klass = Route, options, &block)
-        klass.new(matcher, path_offset, &block).tap do |route|
-          route.parent = self
-          leaf = options.reduce(route) { |acc, (key, val)| acc.public_send(key, val) }
-          children.push(route)
-          Docile.dsl_eval(leaf, &block) if block_given?
+      # @param matcher [Wayfarer::Routing::Matcher, nil]
+      # @param klass [Class<Route>]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      def child_route(matcher, klass: Route, **, &)
+        klass.new(matcher: matcher, parent: self, **, &).tap do |route|
+          children.append(route)
         end
       end
-      # rubocop:enable Style/OptionalArguments
     end
   end
 end

data/lib/wayfarer/routing/hash_stack.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class HashStack
+      EmptyStackError = Class.new(StandardError)
+
+      def self.empty
+        new(Route::EMPTY_PARAMS)
+      end
+
+      def initialize(initial_state)
+        @stack = [initial_state]
+      end
+
+      def push(hash)
+        stack.push(stack.last.dup.merge!(hash))
+      end
+
+      def pop
+        stack.pop || raise(EmptyStackError)
+      end
+
+      def to_h
+        stack.last
+      end
+
+    private
+
+      attr_reader :stack
+    end
+  end
+end

data/lib/wayfarer/routing/matchers/custom.rb

@@ -5,19 +5,22 @@ module Wayfarer
     module Matchers
       class Custom
         include Stringify
+        include EmptyParams
 
         attr_reader :delegate
 
-        def initialize(delegate = proc)
+        def initialize(delegate)
           @delegate = delegate
         end
 
-        def match(url)
-          !!delegate.call(url)
+        def evaluate(path_finder)
+          Wayfarer::Routing::RootRoute.new.tap do |route|
+            delegate.call(route, path_finder.uri, path_finder.task)
+          end
         end
 
-        def params(_)
-          {}
+        def to_h
+          { custom: delegate.class.name }
         end
       end
     end