wayfarer 0.4.7 → 0.4.9

data/lib/wayfarer/networking/selenium.rb

@@ -9,7 +9,7 @@ module Wayfarer
       MOCK_RESPONSE_HEADERS = {}.freeze
 
       def create
-        ::Selenium::WebDriver.for(Wayfarer.config[:selenium][:driver], **options)
+        ::Selenium::WebDriver.for(driver, options)
       end
 
       def destroy(instance)
@@ -29,15 +29,23 @@ module Wayfarer
 
     private
 
-      def options
-        Wayfarer.config[:selenium][:options].merge(http_client: http_client)
+      def driver
+        Wayfarer.config.dig(:selenium, :driver)
       end
 
-      def http_client
-        ::Selenium::WebDriver::Remote::Http::Default.new.tap do |client|
-          client.read_timeout = Wayfarer.config[:selenium][:client_timeout]
-        end
+      def options
+        Wayfarer.config.dig(:selenium, :options)
       end
+
+      # def options
+      #   Wayfarer.config[:selenium][:options].merge(http_client: http_client)
+      # end
+
+      # def http_client
+      #   ::Selenium::WebDriver::Remote::Http::Default.new.tap do |client|
+      #     client.read_timeout = Wayfarer.config[:selenium][:client_timeout]
+      #   end
+      # end
     end
   end
 end

data/lib/wayfarer/networking/strategy.rb

@@ -3,8 +3,12 @@
 module Wayfarer
   module Networking
     module Strategy
-      def renew_on
-        []
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def renew_on
+          []
+        end
       end
 
       def fetch(instance, url)

data/lib/wayfarer/page.rb

@@ -50,8 +50,6 @@ module Wayfarer
       @meta ||= MetaInspector.new(url, document: body, headers: headers, normalize_url: false)
     end
 
-  private
-
     def content_type
       @content_type ||= headers["content-type"]
     end

data/lib/wayfarer/parsing/xml.rb

@@ -9,7 +9,7 @@ module Wayfarer
         case variant
         when :xml then Nokogiri::XML(xml)
         when :html then Nokogiri::HTML(xml)
-        else raise ArgumentError, "Unknown type: #{type}"
+        else raise ArgumentError, "Unknown type: #{variant}"
         end
       end
     end

data/lib/wayfarer/parsing.rb

@@ -8,15 +8,12 @@ module Wayfarer
     # @!visibility private
     FALLBACK_CONTENT_TYPE = "application/octet-stream"
 
-    mattr_accessor :registry, default: { "application/json" => JSON,
-                                         "text/html" => [XML, :html],
-                                         "application/xml" => [XML, :xml] }
-
   module_function
 
     # @!visibility private
     def parse(body, content_type = FALLBACK_CONTENT_TYPE)
-      parser, args = registry[content_type] || return
+      parser, args = Wayfarer.config.dig(:parsing, :content_types, content_type)
+      return unless parser
 
       parser.parse(body, *args)
     end

data/lib/wayfarer/redis/barrier.rb

@@ -2,26 +2,39 @@
 
 module Wayfarer
   module Redis
+    # A Barrier prevents processing the same key more than once.
+    # It marks keys in a Redis hash as they are encountered.
+    # Once a key is added, it cannot get removed.
     class Barrier
       include Resettable
 
+      # @return [Hash] the task configuration, including redis_pool and batch name
       attr_reader :task
 
+      VALUE = ""
+
+      # @param task [Wayfarer::Task] task context
       def initialize(task)
         @task = task
         @redis_pool = task[:redis_pool]
       end
 
+      # @return [String] the Redis key for this barrier
       def redis_key
         "wayfarer-barrier-#{task.batch}"
       end
 
-      def check!(url)
-        !redis_pool.with { |conn| conn.hsetnx(redis_key, url, "") }
+      # Checks if a key has already been passed through the barrier.
+      #
+      # @param key [String] the key to check
+      # @return [Boolean] true if the key has already been seen, false otherwise
+      def check!(key)
+        !redis_pool.with { |conn| conn.hsetnx(redis_key, key, VALUE) }
       end
 
     private
 
+      # @return [ConnectionPool] the Redis connection pool
       attr_reader :redis_pool
     end
   end

data/lib/wayfarer/redis/counter.rb

@@ -7,9 +7,8 @@ module Wayfarer
 
       attr_reader :task
 
-      def initialize(task, &callback)
+      def initialize(task)
         @task = task
-        @callback = callback
         @redis_pool = task[:redis_pool]
       end
 

data/lib/wayfarer/routing/dsl.rb

@@ -2,57 +2,192 @@
 
 module Wayfarer
   module Routing
+    # Routing DSL that declares a tree of {Route}s. Routing trees decide
+    # whether a URL gets processed. Each route has a matcher which is a
+    # predicate. Routes are searched depth-first by a {PathFinder}.
+    #
+    # When you call a DSL method on a route, you declare a child route on the
+    # route. You can pass a keyword list and a block when you call DSL methods.
+    # Keyword lists result in route chains. For example, if you declare:
+    #
+    # ```ruby
+    # route.host "example.com", path: ":foo", query: { page: 1 }
+    # ```
+    #
+    # The last route with the query matcher is returned, and the hierarchy is
+    # `host > path > query`.
+    #
+    # To append multiple child rules, pass a block within which you can call
+    # DSL methods to declare child routes:
+    #
+    # ```ruby
+    # route.host "example.com" do
+    #   path ":foo"
+    #   query page: 1
+    # end
+    # ```
+    #
+    # The query matcher is returned again, but the hierarchy is
+    # `host > path`, `host > query`.
+    #
+    # @see Route
+    # @see PathFinder
     module DSL
-      def url(url, options = {}, &block)
-        add_child_route(Matchers::URL.new(url), path_offset, options, &block)
+      # Match URLs exactly.
+      #
+      # A trailing slash in +url+ is ignored so
+      # "https://example.com" and "https://example.com/" are equivalent.
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param url [String]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Match URL exactly
+      #   route.url("https://www.iana.org/help/example-domains").to(:index)
+      def url(url, **, &)
+        child_route(Matchers::URL.new(url), **, &)
       end
 
-      def host(host, options = {}, &block)
-        add_child_route(Matchers::Host.new(host), path_offset, options, &block)
+      # Match hostnames excluding the port number.
+      #
+      # * `String` is compared literally
+      # * `Regexp` is matched against the host
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param host [String, Regexp]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Literal host
+      #   route.host("example.com").to(:home)
+      # @example Regular expression
+      #   route.host(/example\.com/) do
+      #     path "users/:id", to: :user
+      #   end
+      def host(host, **, &)
+        child_route(Matchers::Host.new(host), **, &)
       end
 
-      def path(path, options = {}, &block)
-        offset = File.join(path_offset, path)
+      # Match and consume path fragments. You can use Sinatra-style pattern
+      # matching to extract data from segments.
+      #
+      # @see https://github.com/sinatra/mustermann/blob/main/mustermann/README.md#-sinatra-pattern
+      #
+      # This matcher doesn't collect `params`.
+      # A leading slash is enforced on the path string.
+      #
+      # @example Capture segment
+      #   route.path(":segment").to(:show)
+      # @example Nested paths
+      #   route.path("/").path("foo").path(":id").to(:detail)
+      # @param path [String]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      def path(path, **, &)
+        path = File.join(File::SEPARATOR, path)
 
-        add_child_route(nil, offset, options, &block).tap do |route|
-          route.matcher = Matchers::Path.new(offset, route)
-        end
-      end
-
-      def query(fields, options = {}, &block)
-        add_child_route(Matchers::Query.new(fields), path_offset, options, &block)
+        child_route(Matchers::Path.new(path), **, &)
       end
 
-      def scheme(scheme, options = {}, &block)
-        add_child_route(Matchers::Scheme.new(scheme), path_offset, options, &block)
+      # Match query parameters.
+      #
+      # Each key/value pair must be present *at least once*; if multiple values
+      # occur the last one wins (like Rack).
+      #
+      # Supported value types:
+      #
+      # * `String` - exact match
+      # * `Regexp` - regular expression match
+      # * `Integer` - exact numeric match
+      # * `Range` - inclusive numeric range
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param fields [Hash{Symbol,String => String,Regexp,Integer,Range}]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Simple parameter
+      #   route.query(foo: "bar").to(:index)
+      # @example Page range
+      #   route.query(page: 5..12).to(:index)
+      def query(fields, **, &)
+        child_route(Matchers::Query.new(fields), **, &)
       end
 
-      def suffix(suffix, options = {}, &block)
-        add_child_route(Matchers::Suffix.new(suffix), path_offset, options, &block)
+      # Match URL schemes (protocols).
+      #
+      # This matcher doesn't collect `params`.
+      #
+      # @param scheme [String, Symbol]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example HTTPS vs HTTP
+      #   route.scheme(:https).to(:tls)
+      #   route.scheme(:http).to(:plain)
+      def scheme(scheme, **, &)
+        child_route(Matchers::Scheme.new(scheme), **, &)
       end
 
-      def to(action, options = {}, &block)
-        add_child_route(Matchers::Custom.new { true }, path_offset, TargetRoute, options, &block).tap do |route|
-          route.action = action
-        end
+      # Declares the action for the current route branch. An action is a symbol
+      # for an instance method, or a {Wayfarer::Handler}.
+      #
+      # In case of conflicting actions for a matching route path, the last
+      # matched action takes precedence.
+      #
+      # @param action [Symbol, Wayfarer::Handler] method or {Handler} to call.
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::TargetRoute]
+      #
+      # @example Last action wins
+      #   route.to(:alpha).to(:beta) # => routes to :beta
+      def to(action, **, &)
+        child_route(nil, action: action, klass: TargetRoute, **, &)
       end
 
-      def custom(delegate, options = {}, &block)
-        add_child_route(Matchers::Custom.new(delegate), path_offset, options, &block)
+      # Match URLs dynamically by declaring a route from a block during route
+      # evaluation. Custom matchers are passed a transient root route which will
+      # be followed. Custom matchers match when their dynamically declared
+      # subtree matches the URL.
+      #
+      # @param options [Hash]
+      # @yield [root, uri, task]
+      # @yieldparam  [Wayfarer::Routing::RootRoute] root route to populate
+      # @yieldparam uri [Addressable::URI] parsed task URL
+      # @yieldparam task [Wayfarer::Task] current task
+      # @return [Wayfarer::Routing::Route]
+      #
+      # @example Batch routing
+      #   route.custom do |root, _uri, task|
+      #     database_record = Crawl.find_by(batch: task.batch)
+      #     root.host(database_record.hostname_to_crawl).to(:index)
+      #   end
+      def custom(**, &block)
+        child_route(Matchers::Custom.new(block), klass: SubRoute, **)
       end
 
     private
 
-      # rubocop:disable Style/OptionalArguments
-      def add_child_route(matcher, path_offset, klass = Route, options, &block)
-        klass.new(matcher, path_offset, &block).tap do |route|
-          route.parent = self
-          leaf = options.reduce(route) { |acc, (key, val)| acc.public_send(key, val) }
-          children.push(route)
-          Docile.dsl_eval(leaf, &block) if block_given?
+      # @param matcher [Wayfarer::Routing::Matcher, nil]
+      # @param klass [Class<Route>]
+      # @param options [Hash]
+      # @yield [route]
+      # @return [Wayfarer::Routing::Route]
+      def child_route(matcher, klass: Route, **, &)
+        klass.new(matcher: matcher, parent: self, **, &).tap do |route|
+          children.append(route)
         end
       end
-      # rubocop:enable Style/OptionalArguments
     end
   end
 end

data/lib/wayfarer/routing/hash_stack.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class HashStack
+      EmptyStackError = Class.new(StandardError)
+
+      def self.empty
+        new(Route::EMPTY_PARAMS)
+      end
+
+      def initialize(initial_state)
+        @stack = [initial_state]
+      end
+
+      def push(hash)
+        stack.push(stack.last.dup.merge!(hash))
+      end
+
+      def pop
+        stack.pop || raise(EmptyStackError)
+      end
+
+      def to_h
+        stack.last
+      end
+
+    private
+
+      attr_reader :stack
+    end
+  end
+end

data/lib/wayfarer/routing/matchers/custom.rb

@@ -5,19 +5,22 @@ module Wayfarer
     module Matchers
       class Custom
         include Stringify
+        include EmptyParams
 
         attr_reader :delegate
 
-        def initialize(delegate = proc)
+        def initialize(delegate)
           @delegate = delegate
         end
 
-        def match(url)
-          !!delegate.call(url)
+        def evaluate(path_finder)
+          Wayfarer::Routing::RootRoute.new.tap do |route|
+            delegate.call(route, path_finder.uri, path_finder.task)
+          end
         end
 
-        def params(_)
-          {}
+        def to_h
+          { custom: delegate.class.name }
         end
       end
     end

data/lib/wayfarer/routing/matchers/{suffix.rb → empty_params.rb}

@@ -3,13 +3,9 @@
 module Wayfarer
   module Routing
     module Matchers
-      Suffix = Struct.new(:suffix) do
-        def match(url)
-          url.path.end_with?(suffix)
-        end
-
+      module EmptyParams
         def params(_)
-          {}
+          Wayfarer::Routing::Route::EMPTY_PARAMS
         end
       end
     end

data/lib/wayfarer/routing/matchers/host.rb

@@ -3,17 +3,23 @@
 module Wayfarer
   module Routing
     module Matchers
-      Host = Struct.new(:host) do
-        # rubocop:disable Style/CaseEquality
-        def match(url)
-          # url.host excludes the port
-          # TODO: Test case, docs
-          host === url.authority
+      class Host
+        include EmptyParams
+
+        attr_reader :host
+
+        def initialize(host)
+          @host = host
+        end
+
+        def evaluate(path_finder)
+          # rubocop:disable Style/CaseEquality -- String and Regexp matching
+          @host === path_finder.uri.host
+          # rubocop:enable Style/CaseEquality
         end
-        # rubocop:enable Style/CaseEquality
 
-        def params(_)
-          {}
+        def to_h
+          { name: host }
         end
       end
     end

data/lib/wayfarer/routing/matchers/path.rb

@@ -4,46 +4,24 @@ module Wayfarer
   module Routing
     module Matchers
       class Path
-        attr_reader :path,
-                    :route,
-                    :peeking,
-                    :matcher
+        PATTERN_TYPE = "sinatra"
 
-        def initialize(path, route)
-          @path = path
-          @route = route
-          @peeking = false
-          @matcher = Mustermann.new(path, type: "sinatra") # TODO: Add type constant
-        end
-
-        def match(url)
-          # TODO: Ditch this and add a parameter to the path DSL#path
-          #
-          # If the route's branch contains other path matchers in child routes,
-          # match the beginning of the path (peeking), instead of the whole path.
-          route.accept(self)
+        attr_reader :pattern
 
-          !!(if peeking
-               matcher.peek(url.path)
-             else
-               matcher.match(url.path)
-             end)
+        def initialize(pattern)
+          @pattern = Mustermann.new(pattern, type: PATTERN_TYPE)
         end
 
-        def params(url)
-          return {} unless match(url)
-
-          matcher.params(url.path) || {}
+        def evaluate(path_finder)
+          path_finder.path_consumer.current_state.valid?
         end
 
-        def visit(route)
-          return true if route == self.route
-          return true if route.is_a?(TargetRoute)
-
-          return unless route.matcher.is_a?(self.class)
+        def params(path_finder)
+          path_finder.path_consumer.current_state.params
+        end
 
-          @peeking = true
-          false
+        def to_h
+          { pattern: pattern.to_s }
         end
       end
     end

data/lib/wayfarer/routing/matchers/query.rb

@@ -3,33 +3,56 @@
 module Wayfarer
   module Routing
     module Matchers
-      Query = Struct.new(:fields) do
-        def match(url)
-          query = url.query
+      class Query
+        attr_reader :fields
+
+        def initialize(fields)
+          @fields = fields
+        end
+
+        def evaluate(path_finder)
+          query = path_finder.uri.query
 
-          # CGI::parse throws a NoMethodError if the query is an empty string
           return false if query.nil? || query.empty?
 
-          CGI.parse(query).none? { |field, vals| violates?(field, vals) }
-        end
+          # TODO: Move query parameter parsing to PathFinder
+          parsed_query = CGI.parse(query)
 
-        def params(url)
-          return {} unless match(url)
+          # For every expected field in `fields`,
+          # check that it is present and does not violate the constraint.
+          fields.all? do |(field_key, constraint)|
+            param_vals = parsed_query[field_key.to_s]
 
-          CGI.parse(url.query)
-             .select { |(k, _)| fields.keys.include?(k.to_sym) }
-             .transform_values(&:last)
+            # If the param is absent, mismatch
+            if param_vals.nil? || param_vals.empty?
+              false
+            else
+              !violates_constraint?(constraint, param_vals)
+            end
+          end
         end
 
-      private
+        def params(path_finder)
+          return {} unless evaluate(path_finder)
 
-        # rubocop:disable Lint/AssignmentInCondition
-        def violates?(field, vals)
-          return false unless constraint = fields[field.to_sym]
+          parsed_query = CGI.parse(path_finder.uri.query)
 
-          violates_constraint?(constraint, vals)
+          parsed_query
+            .select { |k, _| fields.keys.include?(k.to_sym) }
+            .transform_values(&:last)
         end
-        # rubocop:enable Lint/AssignmentInCondition
+
+        def to_h
+          fields.transform_values do |val|
+            case val
+            when Regexp then val.inspect
+            when Range then { min: val.min, max: val.max }
+            else val
+            end
+          end
+        end
+
+      private
 
         def violates_constraint?(constraint, vals)
           case constraint
@@ -37,6 +60,7 @@ module Wayfarer
           when Integer then violates_integer?(constraint, vals)
           when Regexp  then violates_regexp?(constraint, vals)
           when Range   then violates_range?(constraint, vals)
+          else false
           end
         end
 

data/lib/wayfarer/routing/matchers/result.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    module Matchers
+      module Result
+        Match = Class.new.include(Singleton)
+        Mismatch = Class.new.include(Singleton)
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/matchers/scheme.rb

@@ -3,13 +3,21 @@
 module Wayfarer
   module Routing
     module Matchers
-      Scheme = Struct.new(:scheme) do
-        def match(url)
-          url.scheme == scheme.to_s
+      class Scheme
+        include EmptyParams
+
+        attr_reader :scheme
+
+        def initialize(scheme)
+          @scheme = scheme
+        end
+
+        def evaluate(path_finder)
+          path_finder.uri.scheme == scheme.to_s
         end
 
-        def params(_)
-          {}
+        def to_h
+          { scheme: scheme }
         end
       end
     end