wayfarer 0.4.5 → 0.4.7

data/lib/wayfarer/middleware/dispatch.rb

@@ -6,12 +6,20 @@ module Wayfarer
       extend Base
 
       def call(task)
-        controller = task.metadata.controller
+        controller = task[:controller]
 
-        controller.run_callbacks(:action) do
-          case action = task.metadata.action
+        task[:return_value] = controller.run_callbacks(:action) do
+          case action = task[:action]
           when Symbol then controller.public_send(action)
-          else action.new.call(task)
+          when Array
+            handler, method = action
+            task[:action] = method
+            handler.new.call(task)
+          else
+            raise ArgumentError, "invalid action: #{action.inspect}" unless action&.include?(Wayfarer::Handler)
+
+            task[:action] = nil # TODO: Test
+            action.new.call(task)
           end
         end
 

data/lib/wayfarer/middleware/normalize.rb

@@ -4,23 +4,24 @@ module Wayfarer
   module Middleware
     class Normalize
       extend Base
+      include Wayfarer::Logging.emit(
+        invalid: [:info, "Failed to normalize HTTP(S) URL"]
+      )
 
-      def call(task)
-        yield if block_given?
+      def self.normalize(uri)
+        return uri.to_s unless %w[http https].include?(uri.scheme)
 
-        task.metadata.staged_urls = SortedSet.new(normalized_urls(task).compact)
+        NormalizeUrl.process(uri)
+      rescue NormalizeUrl::InvalidURIError
+        nil
       end
 
-    private
+      def call(task)
+        return (yield if block_given?) if task[:normalized_url]
 
-      def normalized_urls(task)
-        task.metadata.staged_urls.map(&method(:normalize))
-      end
+        return log(:invalid, task) unless (task[:normalized_url] = self.class.normalize(task[:uri]))
 
-      def normalize(url)
-        NormalizeUrl.process(url)
-      rescue NormalizeUrl::InvalidURIError
-        nil
+        yield if block_given?
       end
     end
   end

data/lib/wayfarer/middleware/redis.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Redis
+      extend Base
+
+      def call(task)
+        task[:redis_pool] ||= Wayfarer::Redis::Pool.instance
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/router.rb

@@ -5,51 +5,49 @@ module Wayfarer
     class Router
       extend Base
 
-      module API
-        def self.included(base)
-          base.extend(ClassMethods)
-          base.include(InstanceMethods)
-        end
-
-        module ClassMethods
-          def router
-            # TODO: Use cattr_accessor
-            @router ||= Wayfarer::Routing::Router.new
-          end
+      include Wayfarer::Logging.emit(
+        mismatch: [:info, "No matching route"],
+        match: [:info, "Routing to %<action>s"],
+        already_routed: [:debug, "Already routed to %<action>s"]
+      )
 
-          def route(&block)
-            router.draw(&block) if block_given?
-          end
+      module API
+        extend ActiveSupport::Concern
 
-          def steer(&block)
-            define_method(:steer) { block.call(task) }
-          end
+        included do
+          class_attribute :route,
+                          default: Wayfarer::Routing::RootRoute.new,
+                          instance_accessor: false,
+                          instance_predicate: false
         end
 
-        module InstanceMethods
-          def steer
-            []
-          end
+        def action
+          task[:action]
+        end
 
-          def params
-            task.metadata.params
-          end
+        def params
+          task[:params]
         end
       end
 
       def call(task)
-        controller = task.metadata.controller
-        # TODO: The router has to be cloned because it's not thread-safe
-        router = controller.class.router.clone
-        url = Addressable::URI.parse(task.url)
-
-        case result = router.invoke(url, controller.steer)
-        when Routing::Result::Mismatch
-          return
+        # Avoid rerouting when dispatching a [Controller, :action] pair
+        if (action = task[:action])
+          log(:already_routed, task, action: action)
+
+          return (yield if block_given?)
+        end
+
+        case result = task[:controller].class.route.invoke(task[:uri])
+        when Routing::Result::Mismatch then return log(:mismatch, task)
         when Routing::Result::Match
-          task.metadata.action = result.action
-          task.metadata.params ||= ActiveSupport::HashWithIndifferentAccess.new
-          task.metadata.params.merge!(result.params)
+          action = result.action
+
+          log(:match, task, action: action.inspect)
+
+          task[:action] = action
+          task[:params] ||= ActiveSupport::HashWithIndifferentAccess.new
+          task[:params].merge!(result.params)
         end
 
         yield if block_given?

data/lib/wayfarer/middleware/stage.rb

@@ -7,20 +7,20 @@ module Wayfarer
 
       module API
         def stage(urls)
-          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url.to_s) }
+          Array.wrap(urls).each { |url| task[:staged_urls].add(url.to_s) }
         end
       end
 
       def call(task)
-        task.metadata.staged_urls = SortedSet.new
+        task[:staged_urls] = Set.new
 
         yield if block_given?
 
-        task.metadata.staged_urls.each do |url|
-          task.metadata.job.class.crawl(url, batch: task.batch)
+        task[:staged_urls].each do |url|
+          task[:job].class.crawl(url, batch: task.batch)
         end
 
-        task.metadata.staged_urls.clear
+        task[:staged_urls].clear
       end
     end
   end

data/lib/wayfarer/middleware/uri_parser.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class UriParser
+      extend Base
+
+      include Wayfarer::Logging.emit(
+        invalid: [:info, "Not processing invalid URL (%<message>s)"]
+      )
+
+      module API
+        def uri
+          task[:uri]
+        end
+      end
+
+      def call(task)
+        # TODO: Test
+        task[:uri] ||= begin
+          Addressable::URI.parse(task.url).normalize
+        rescue Addressable::URI::InvalidURIError => e
+          return log(:invalid, task, message: e.message)
+        end
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/user_agent.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class UserAgent
+      extend Base
+
+      module API
+        def user_agent
+          task[:context]&.instance
+        end
+
+        def page(live: false)
+          return task[:page] unless live
+
+          task[:page] = task[:context].live&.page || task[:page]
+        end
+
+        def fetch(url, follow: 3)
+          (@http ||= Wayfarer::Networking::Follow.http).fetch(url, follow: follow)
+        end
+      end
+
+      def call(task)
+        pool.with do |context|
+          task[:context] = context
+
+          result = task[:controller].run_callbacks(:fetch) do
+            context.fetch(task.url)
+          end
+
+          case result
+          when Networking::Result::Redirect
+            task[:controller].stage(result.redirect_url)
+          when Networking::Result::Success
+            task[:page] = result.page
+            yield if block_given?
+          end
+        end
+      end
+
+    private
+
+      def pool
+        Wayfarer::Networking::Pool.instance
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/capybara.rb

@@ -6,7 +6,7 @@ module Wayfarer
       include Strategy
 
       def create
-        ::Capybara::Session.new(Wayfarer.config.capybara.driver, nil)
+        ::Capybara::Session.new(Wayfarer.config[:capybara][:driver], nil)
       end
 
       def destroy(instance)

data/lib/wayfarer/networking/context.rb

@@ -25,8 +25,8 @@ module Wayfarer
 
       def supervise
         yield
-      rescue *strategy.renew_on => e
-        renew
+      rescue *strategy.renew_on, *Wayfarer.config[:network][:renew_on] => e
+        renew # may raise
       ensure
         # If renewing raises, re-raise the originally caught exception
         # TODO: Not nice this effectively swallows exceptions

data/lib/wayfarer/networking/ferrum.rb

@@ -10,8 +10,8 @@ module Wayfarer
       end
 
       def create
-        ::Ferrum::Browser.new(Wayfarer.config.ferrum.options).tap do |browser|
-          browser.headers.set(Wayfarer.config.network.http_headers)
+        ::Ferrum::Browser.new(Wayfarer.config[:ferrum][:options]).tap do |browser|
+          browser.headers.set(Wayfarer.config[:network][:http_headers])
         end
       end
 

data/lib/wayfarer/networking/follow.rb

@@ -2,15 +2,21 @@
 
 module Wayfarer
   module Networking
-    RedirectsExhaustedError = Class.new(StandardError)
+    class Follow
+      RedirectsExhaustedError = Class.new(StandardError)
 
-    Follow = Struct.new(:context) do
-      extend Forwardable
+      def self.http
+        new(Wayfarer::Networking::Context.new(Wayfarer::Networking::HTTP.new))
+      end
+
+      attr_reader :context
 
-      delegate %i[live renew instance] => :context
+      def initialize(context)
+        @context = context
+      end
 
-      def fetch(url, follow: 3)
-        raise RedirectsExhaustedError if follow.negative?
+      def fetch(url, follow:)
+        raise RedirectsExhaustedError if follow < 0
 
         case result = context.fetch(url)
         when Result::Success then result.page

data/lib/wayfarer/networking/http.rb

@@ -9,7 +9,7 @@ module Wayfarer
 
       def create
         Net::HTTP::Persistent.new(name: CONNECTION_NAME).tap do |conn|
-          Wayfarer.config.network.http_headers.each do |key, val|
+          Wayfarer.config[:network][:http_headers].each do |key, val|
             conn.override_headers[key] = val
           end
         end

data/lib/wayfarer/networking/pool.rb

@@ -5,25 +5,30 @@ module Wayfarer
     class Pool
       include Singleton
 
-      cattr_accessor :registry, default: { http: HTTP,
-                                           ferrum: Ferrum,
-                                           selenium: Selenium,
-                                           capybara: Capybara }
-
-      def pool
-        @pool ||= ConnectionPool.new(size: Wayfarer.config.network.pool_size,
-                                     timeout: Wayfarer.config.network.pool_timeout,
-                                     &method(:context))
+      class_attribute :registry,
+                      default: { http: HTTP,
+                                 ferrum: Ferrum,
+                                 selenium: Selenium,
+                                 capybara: Capybara },
+                      instance_accessor: false,
+                      instance_predicate: false
+
+      def initialize
+        @pool = ConnectionPool.new(size: Wayfarer.config[:network][:pool_size],
+                                   timeout: Wayfarer.config[:network][:pool_timeout],
+                                   &method(:context))
+
+        at_exit { free }
       end
 
       def with(&block)
-        pool.with(&block)
+        @pool.with(&block)
       rescue ConnectionPool::TimeoutError => e
         raise Wayfarer::UserAgentTimeoutError, e
       end
 
       def free
-        pool.shutdown(&:renew)
+        @pool.shutdown(&:renew)
       end
 
     private
@@ -33,7 +38,7 @@ module Wayfarer
       end
 
       def strategy
-        self.class.registry[Wayfarer.config.network.agent].new
+        self.class.registry[Wayfarer.config[:network][:agent]].new
       end
     end
   end

data/lib/wayfarer/networking/selenium.rb

@@ -9,7 +9,7 @@ module Wayfarer
       MOCK_RESPONSE_HEADERS = {}.freeze
 
       def create
-        ::Selenium::WebDriver.for(Wayfarer.config.selenium.driver, **options)
+        ::Selenium::WebDriver.for(Wayfarer.config[:selenium][:driver], **options)
       end
 
       def destroy(instance)
@@ -30,12 +30,12 @@ module Wayfarer
     private
 
       def options
-        Wayfarer.config.selenium.options.merge(http_client: http_client)
+        Wayfarer.config[:selenium][:options].merge(http_client: http_client)
       end
 
       def http_client
         ::Selenium::WebDriver::Remote::Http::Default.new.tap do |client|
-          client.read_timeout = Wayfarer.config.selenium.client_timeout
+          client.read_timeout = Wayfarer.config[:selenium][:client_timeout]
         end
       end
     end

data/lib/wayfarer/networking/strategy.rb

@@ -13,13 +13,13 @@ module Wayfarer
       end
 
       def navigate(_instance, _url)
-        raise NoMethodError
+        raise NotImplementedError
       end
 
       def live(_instance); end
 
       def create
-        raise NoMethodError
+        raise NotImplementedError
       end
 
       def destroy(_instance); end

data/lib/wayfarer/page.rb

@@ -1,12 +1,22 @@
 # frozen_string_literal: true
 
 module Wayfarer
+  # @!attribute [r] url
+  #   @return [String] the URL that was fetched
+  # @!attribute [r] status_code
+  #   @return [Fixnum] HTTP status code
+  # @!attribute [r] body
+  #   @return [String] the body of the response
+  # @!attribute [r] headers
+  #   @return [Hash] the headers of the response
+  #   @note HTTP header keys are downcased, for example: `content-type`.
   class Page
     attr_reader :url,
                 :status_code,
                 :body,
                 :headers
 
+    # @!visibility private
     def initialize(url:, status_code:, body:, headers:)
       @url = url
       @status_code = status_code
@@ -14,24 +24,36 @@ module Wayfarer
       @headers = headers.transform_keys(&:downcase)
     end
 
-    def doc
-      return @doc if @doc
-
-      # If no Content-Type field is present, assume HTML/XML
-      return @doc = Wayfarer::Parsing::XML.parse_html(body) unless headers["content-type"]
-
-      content_type = headers["content-type"]
-      sub_type = MIME::Types[content_type].first.sub_type
+    # Returns the MIME type of the response.
+    # @return [MIME::Type]
+    # @see https://www.rubydoc.info/gems/mime-types/MIME/Type
+    def mime_type
+      @mime_type ||= MIME::Types[content_type]&.first
+    end
 
-      @doc = case sub_type
-             when "json" then Wayfarer::Parsing::JSON.parse(body)
-             when "xml" then Wayfarer::Parsing::XML.parse_xml(body)
-             else Wayfarer::Parsing::XML.parse_html(body)
-             end
+    # Returns a parsed representation of the HTTP response or the browser DOM,
+    # depending on the Content-Type.
+    # @return [Nokogiri::HTML::Document] when Content-Type is `text/html`
+    # @see https://www.rubydoc.info/github/sparklemotion/nokogiri/Nokogiri/HTML/Document Nokogiri::HTML::Document
+    # @return [Nokogiri::XML::Document] when Content-Type is `text/xml`
+    # @see https://www.rubydoc.info/github/sparklemotion/nokogiri/Nokogiri/XML/Document Nokogiri::XML::Document
+    # @return [Hash] when Content-Type is `application/json`
+    # @note You can register custom parsers with {Wayfarer::Parsing.registry}.
+    def doc
+      @doc ||= Wayfarer::Parsing.parse(body, mime_type&.content_type || content_type)
     end
 
+    # Returns a `MetaInspector::Document`.
+    # @return [MetaInspector::Document]
+    # @see https://www.rubydoc.info/gems/metainspector/MetaInspector/Document
     def meta
-      @meta ||= MetaInspector.new(url, document: body)
+      @meta ||= MetaInspector.new(url, document: body, headers: headers, normalize_url: false)
+    end
+
+  private
+
+    def content_type
+      @content_type ||= headers["content-type"]
     end
   end
 end

data/lib/wayfarer/parsing/xml.rb

@@ -5,12 +5,12 @@ module Wayfarer
     module XML
     module_function
 
-      def parse_xml(xml)
-        Nokogiri::XML(xml)
-      end
-
-      def parse_html(html)
-        Nokogiri::HTML(html)
+      def parse(xml, variant)
+        case variant
+        when :xml then Nokogiri::XML(xml)
+        when :html then Nokogiri::HTML(xml)
+        else raise ArgumentError, "Unknown type: #{type}"
+        end
       end
     end
   end

data/lib/wayfarer/parsing.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  # @!scope class
+  # @!attribute [r] registry
+  #   @return [Hash] Mapping of Content-Type to parser.
+  module Parsing
+    # @!visibility private
+    FALLBACK_CONTENT_TYPE = "application/octet-stream"
+
+    mattr_accessor :registry, default: { "application/json" => JSON,
+                                         "text/html" => [XML, :html],
+                                         "application/xml" => [XML, :xml] }
+
+  module_function
+
+    # @!visibility private
+    def parse(body, content_type = FALLBACK_CONTENT_TYPE)
+      parser, args = registry[content_type] || return
+
+      parser.parse(body, *args)
+    end
+  end
+end

data/lib/wayfarer/redis/barrier.rb

@@ -2,35 +2,27 @@
 
 module Wayfarer
   module Redis
-    Barrier = Struct.new(:batch) do
-      include Connection
+    class Barrier
+      include Resettable
 
-      def redis_key
-        "wayfarer-barrier-#{batch}"
-      end
+      attr_reader :task
 
-      def reset!
-        redis { |conn| conn.del(redis_key) }
+      def initialize(task)
+        @task = task
+        @redis_pool = task[:redis_pool]
       end
 
-      def seen?(url)
-        !redis { |conn| conn.sadd(redis_key, url) }
+      def redis_key
+        "wayfarer-barrier-#{task.batch}"
       end
 
-      def peek(urls)
-        major, minor, = Version.determine
-
-        # SMISMEMBER is only supported on Redis >= 6.2.0
-        if major > 6 || (major == 6 && minor >= 2)
-          redis { |conn| conn.smismember(redis_key, urls) }.map { |val| val == 1 }
-        else
-          urls.map { |url| redis { |conn| conn.sismember(redis_key, url) } }
-        end
+      def check!(url)
+        !redis_pool.with { |conn| conn.hsetnx(redis_key, url, "") }
       end
 
-      def unsee(url)
-        redis { |conn| conn.srem(redis_key, url) }
-      end
+    private
+
+      attr_reader :redis_pool
     end
   end
 end

data/lib/wayfarer/redis/counter.rb

@@ -2,28 +2,38 @@
 
 module Wayfarer
   module Redis
-    Counter = Struct.new(:batch) do
-      include Connection
+    class Counter
+      include Resettable
 
-      def redis_key
-        "wayfarer-counter-#{batch}"
+      attr_reader :task
+
+      def initialize(task, &callback)
+        @task = task
+        @callback = callback
+        @redis_pool = task[:redis_pool]
       end
 
-      def reset!
-        redis { |conn| conn.del(redis_key) }
+      def redis_key
+        "wayfarer-counter-#{@task.batch}"
       end
 
       def value
-        redis { |conn| conn.get(redis_key) }.to_i
+        redis_pool.with { |conn| conn.get(redis_key) }.to_i
       end
 
       def increment
-        redis { |conn| conn.incr(redis_key) }
+        redis_pool.with { |conn| conn.incr(redis_key) }
       end
 
       def decrement
-        redis { |conn| conn.decr(redis_key) }
+        redis_pool.with { |conn| conn.decr(redis_key) }.tap do |val|
+          @callback&.call if val == 0
+        end
       end
+
+    private
+
+      attr_reader :redis_pool
     end
   end
 end

data/lib/wayfarer/redis/pool.rb

@@ -10,7 +10,7 @@ module Wayfarer
 
       def initialize
         @pool = ConnectionPool.new do
-          Wayfarer.config.redis.factory.call(Wayfarer.config.redis)
+          Wayfarer.config[:redis][:factory].call(Wayfarer.config[:redis])
         end
       end