wayfarer 0.4.6 → 0.4.8

data/lib/wayfarer/handler.rb

@@ -1,15 +1,23 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  class Handler
-    include Wayfarer::Middleware::Controller
+  module Handler
+    extend ActiveSupport::Concern
 
-    api Wayfarer::Middleware::Fetch
-    api Wayfarer::Middleware::Stage
+    included do
+      include Wayfarer::Middleware::Controller
 
-    use Wayfarer::Middleware::Router
-    use Wayfarer::Middleware::Dispatch
+      use Wayfarer::Middleware::ContentType
+      use Wayfarer::Middleware::Router
+      use Wayfarer::Middleware::Dispatch
 
-    singleton_class.undef_method :after_batch
+      api Wayfarer::Middleware::UserAgent
+      api Wayfarer::Middleware::Stage
+
+      singleton_class.undef_method :before_fetch
+      singleton_class.undef_method :around_fetch
+      singleton_class.undef_method :after_fetch
+      singleton_class.undef_method :after_batch
+    end
   end
 end

data/lib/wayfarer/kv.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  # Provides a key-value store via `[]` and `[]=`.
+  #
+  # @api private
+  module KV
+    # @param key [Object] key to fetch
+    # @return [Object, nil] value associated with the key or `nil`
+    def [](key)
+      kv[key]
+    end
+
+    # @param key [Object] key to set
+    # @param value [Object] value to set
+    # @return [Object] value that was set
+    def []=(key, value)
+      kv[key] = value
+    end
+
+  private
+
+    # @return [Hash<Object, Object>]
+    def kv
+      @kv ||= {}
+    end
+  end
+end

data/lib/wayfarer/logging.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Logging
+    mattr_accessor :logger, default: ActiveSupport::Logger.new($stdout)
+
+    def self.emit(...)
+      Emitter.new(...)
+    end
+
+    class Emitter < Module
+      def initialize(messages)
+        @messages = messages
+
+        super()
+      end
+
+      def included(base)
+        messages = @messages
+
+        base.class_eval do
+          define_method(:log) do |key, task, **args|
+            level, msg = messages[key] || raise(ArgumentError, "No log message for #{key.inspect}")
+            severity = ActiveSupport::Logger::Severity.const_get(level.upcase)
+
+            ActiveSupport::TaggedLogging
+              .new(Logging.logger)
+              .tagged(task.batch, task.url, task[:controller]&.class&.name) do |logger|
+              logger.add(severity, msg % args)
+            end
+          end
+        end
+      end
+    end
+
+    private_constant :Emitter
+  end
+end

data/lib/wayfarer/middleware/base.rb

@@ -3,6 +3,8 @@
 module Wayfarer
   module Middleware
     module Base
+      extend ActiveSupport::Concern
+
       API_MODULE = :API
 
       def api

data/lib/wayfarer/middleware/batch_completion.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class BatchCompletion
+      extend Base
+
+      def call(task)
+        # Comparing to the initial state of `exception_executions` allows
+        # us to determine if an exception occurred when the job was performed,
+        # since the `perform.active_job` event is emitted for both successful
+        # and raising jobs.
+        task[:initial_exception_executions] ||= task[:job].exception_executions.clone
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/chain.rb

@@ -2,9 +2,15 @@
 
 module Wayfarer
   module Middleware
-    Chain = Struct.new(:middlewares) do
+    class Chain
       extend Forwardable
 
+      attr_reader :middlewares
+
+      def initialize(middlewares)
+        @middlewares = middlewares
+      end
+
       def self.empty
         new([])
       end

data/lib/wayfarer/middleware/content_type.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class ContentType
+      extend Base
+
+      module API
+        extend ActiveSupport::Concern
+
+        included do
+          class_attribute :allowed_content_types,
+                          default: { index: {}, patterns: Set.new },
+                          instance_accessor: false,
+                          instance_predicate: false
+        end
+
+        class_methods do
+          def content_type(*types)
+            grouped_types = types.group_by(&:class)
+
+            new_index = grouped_types.fetch(String, []).index_with { true }
+            new_patterns = grouped_types.fetch(Regexp, [])
+
+            self.allowed_content_types = {
+              index: allowed_content_types.fetch(:index).merge(new_index),
+              patterns: allowed_content_types.fetch(:patterns).merge(new_patterns)
+            }
+          end
+        end
+      end
+
+      def call(task)
+        yield if block_given? && permitted?(task)
+      end
+
+    private
+
+      def permitted?(task)
+        job = task[:controller]
+        rules = job.class.allowed_content_types
+
+        return true if no_restrictions?(rules)
+
+        return false unless (content_type = task[:page].mime_type&.to_s || task[:page].content_type)
+
+        content_type && matches_rules?(content_type, rules)
+      end
+
+      def matches_rules?(type, rules)
+        rules.fetch(:index).key?(type) || rules.fetch(:patterns).any? { |p| p.match?(type) }
+      end
+
+      def no_restrictions?(rules)
+        rules.values_at(:index, :patterns).all?(&:empty?)
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/controller.rb

@@ -3,16 +3,20 @@
 module Wayfarer
   module Middleware
     module Controller
-      def self.included(base)
-        base.cattr_accessor :chain, default: Chain.empty
-        base.attr_accessor :task
+      extend ActiveSupport::Concern
 
-        base.extend(ClassMethods)
-        base.include(InstanceMethods)
-        base.include(Wayfarer::Callbacks)
+      included do
+        class_attribute :chain,
+                        default: Chain.empty,
+                        instance_accessor: false,
+                        instance_predicate: false
+
+        attr_accessor :task
+
+        include Wayfarer::Callbacks
       end
 
-      module ClassMethods
+      class_methods do
         def use(middleware)
           chain.push(middleware.lazy)
           api(middleware)
@@ -23,17 +27,17 @@ module Wayfarer
         end
       end
 
-      module InstanceMethods
-        def call(task)
-          self.task = task
+      def call(task)
+        self.task = task
 
-          task.metadata.job ||= self
-          task.metadata.controller = self
+        task[:job] ||= self
+        task[:controller] = self
 
-          self.class.chain.call(task) do
-            yield if block_given?
-          end
+        self.class.chain.call(task) do
+          yield if block_given?
         end
+
+        task[:return_value]
       end
     end
   end

data/lib/wayfarer/middleware/dedup.rb

@@ -5,25 +5,34 @@ module Wayfarer
     class Dedup
       extend Base
 
+      include Wayfarer::Logging.emit(
+        deduplicated: [:info, "Deduplicated URL"],
+        retry: [:debug, "Not deduplicating retry"],
+        rerouted: [:debug, "Not deduplicating rerouted task"]
+      )
+
       def call(task)
-        # Was task routed by a previous controller already?
-        return yield if task.metadata.action
+        task[:barrier] ||= Wayfarer::Redis::Barrier.new(task)
 
-        return if task.barrier.seen?(task.url)
+        if task[:job].executions > 1
+          log(:retry, task)
+          return (yield if block_given?)
+        end
 
-        begin
-          yield if block_given?
-        rescue StandardError => e
-          task.barrier.unsee(task.url)
-          raise e
+        if task[:job] != task[:controller]
+          log(:rerouted, task)
+          return (yield if block_given?)
         end
 
-        staged_urls = task.metadata.staged_urls
-        return if staged_urls.none?
+        return log(:deduplicated, task) if task[:barrier].check!(key(task))
+
+        yield if block_given?
+      end
+
+    private
 
-        inclusion = task.barrier.peek(staged_urls.to_a)
-        unseen = staged_urls.zip(inclusion).reject { |_, seen| seen }.map(&:first)
-        task.metadata.staged_urls = SortedSet.new(unseen)
+      def key(task)
+        Wayfarer.config.dig(:deduplication, :key).call(task)
       end
     end
   end

data/lib/wayfarer/middleware/dispatch.rb

@@ -5,13 +5,26 @@ module Wayfarer
     class Dispatch
       extend Base
 
+      InvalidTargetError = Class.new(ArgumentError)
+
       def call(task)
-        controller = task.metadata.controller
+        controller = task[:controller]
 
-        controller.run_callbacks(:action) do
-          case action = task.metadata.action
+        task[:return_value] = controller.run_callbacks(:action) do
+          case action = task[:action]
           when Symbol then controller.public_send(action)
-          else action.new.call(task)
+          when Array
+            handler, method = action
+            task[:action] = method
+            handler.new.call(task)
+          else
+            unless action&.include?(Wayfarer::Handler)
+              raise InvalidTargetError, "routed to invalid action: #{action.inspect}"
+            end
+
+            task[:action] = nil
+
+            action.new.call(task)
           end
         end
 

data/lib/wayfarer/middleware/normalize.rb

@@ -4,23 +4,16 @@ module Wayfarer
   module Middleware
     class Normalize
       extend Base
+      include Wayfarer::Logging.emit(
+        invalid: [:info, "Failed to normalize URL"]
+      )
 
       def call(task)
-        yield if block_given?
-
-        task.metadata.staged_urls = SortedSet.new(normalized_urls(task).compact)
-      end
-
-    private
+        Wayfarer::URI::Normalization.canonical!(task[:uri])
 
-      def normalized_urls(task)
-        task.metadata.staged_urls.map(&method(:normalize))
-      end
-
-      def normalize(url)
-        NormalizeUrl.process(url)
-      rescue NormalizeUrl::InvalidURIError
-        nil
+        yield if block_given?
+      rescue Wayfarer::URI::Normalization::InvalidURIError
+        log(:invalid, task)
       end
     end
   end

data/lib/wayfarer/middleware/redis.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Redis
+      extend Base
+
+      def call(task)
+        task[:redis_pool] ||= Wayfarer::Redis::Pool.instance
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/router.rb

@@ -5,51 +5,49 @@ module Wayfarer
     class Router
       extend Base
 
-      module API
-        def self.included(base)
-          base.extend(ClassMethods)
-          base.include(InstanceMethods)
-        end
-
-        module ClassMethods
-          def router
-            # TODO: Use cattr_accessor
-            @router ||= Wayfarer::Routing::Router.new
-          end
+      include Wayfarer::Logging.emit(
+        mismatch: [:info, "No matching route"],
+        match: [:info, "Routing to %<action>s"],
+        already_routed: [:debug, "Already routed to %<action>s"]
+      )
 
-          def route(&block)
-            router.draw(&block) if block_given?
-          end
+      module API
+        extend ActiveSupport::Concern
 
-          def steer(&block)
-            define_method(:steer) { block.call(task) }
-          end
+        included do
+          class_attribute :route,
+                          default: Wayfarer::Routing::RootRoute.new,
+                          instance_accessor: false,
+                          instance_predicate: false
         end
 
-        module InstanceMethods
-          def steer
-            []
-          end
+        def action
+          task[:action]
+        end
 
-          def params
-            task.metadata.params
-          end
+        def params
+          task[:params]
         end
       end
 
       def call(task)
-        controller = task.metadata.controller
-        # TODO: The router has to be cloned because it's not thread-safe
-        router = controller.class.router.clone
-        url = Addressable::URI.parse(task.url)
-
-        case result = router.invoke(url, controller.steer)
-        when Routing::Result::Mismatch
-          return
+        # Avoid rerouting when dispatching a [Controller, :action] pair
+        if (action = task[:action])
+          log(:already_routed, task, action: action)
+
+          return (yield if block_given?)
+        end
+
+        case result = task[:controller].class.route.invoke(task)
+        when Routing::Result::Mismatch then return log(:mismatch, task)
         when Routing::Result::Match
-          task.metadata.action = result.action
-          task.metadata.params ||= ActiveSupport::HashWithIndifferentAccess.new
-          task.metadata.params.merge!(result.params)
+          action = result.action
+
+          log(:match, task, action: action.inspect)
+
+          task[:action] = action
+          task[:params] ||= ActiveSupport::HashWithIndifferentAccess.new
+          task[:params].merge!(result.params)
         end
 
         yield if block_given?

data/lib/wayfarer/middleware/stage.rb

@@ -7,20 +7,20 @@ module Wayfarer
 
       module API
         def stage(urls)
-          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url.to_s) }
+          Array.wrap(urls).each { |url| task[:staged_urls].add(url.to_s) }
         end
       end
 
       def call(task)
-        task.metadata.staged_urls = SortedSet.new
+        task[:staged_urls] = Set.new
 
         yield if block_given?
 
-        task.metadata.staged_urls.each do |url|
-          task.metadata.job.class.crawl(url, batch: task.batch)
+        task[:staged_urls].each do |url|
+          task[:job].class.crawl(url, batch: task.batch)
         end
 
-        task.metadata.staged_urls.clear
+        task[:staged_urls].clear
       end
     end
   end

data/lib/wayfarer/middleware/uri_parser.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class UriParser
+      extend Base
+
+      include Wayfarer::Logging.emit(
+        invalid: [:info, "Not processing invalid URL (%<message>s)"]
+      )
+
+      module API
+        def uri
+          task[:uri]
+        end
+      end
+
+      def call(task)
+        return (yield if block_given?) if task[:uri]
+
+        begin
+          task[:uri] = Addressable::URI.parse(task.url)
+        rescue Addressable::URI::InvalidURIError => e
+          return log(:invalid, task, message: e.message)
+        end
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/user_agent.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class UserAgent
+      extend Base
+
+      module API
+        def user_agent
+          task[:context]&.instance
+        end
+
+        def page(live: false)
+          return task[:page] unless live
+
+          task[:page] = task[:context].live&.page || task[:page]
+        end
+
+        def fetch(url, follow: 3)
+          (@http ||= Wayfarer::Networking::Follow.http).fetch(url, follow: follow)
+        end
+      end
+
+      def call(task)
+        pool.with do |context|
+          task[:context] = context
+
+          result = task[:controller].run_callbacks(:fetch) do
+            context.fetch(task.url)
+          end
+
+          case result
+          when Networking::Result::Redirect
+            task[:controller].stage(result.redirect_url)
+          when Networking::Result::Success
+            task[:page] = result.page
+            yield if block_given?
+          end
+        end
+      end
+
+    private
+
+      def pool
+        Wayfarer::Networking::Pool.instance
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/capybara.rb

@@ -6,7 +6,7 @@ module Wayfarer
       include Strategy
 
       def create
-        ::Capybara::Session.new(Wayfarer.config.capybara.driver, nil)
+        ::Capybara::Session.new(Wayfarer.config[:capybara][:driver], nil)
       end
 
       def destroy(instance)

data/lib/wayfarer/networking/context.rb

@@ -2,11 +2,22 @@
 
 module Wayfarer
   module Networking
-    Context = Struct.new(:strategy) do
+    class Context
+      attr_reader :strategy
+
+      def initialize(strategy)
+        @strategy = strategy
+      end
+
       def fetch(url)
         supervise { strategy.fetch(instance, url) }
       end
 
+      def navigate(url)
+        fetch(url)
+        live
+      end
+
       def live
         supervise { strategy.live(instance) }
       end
@@ -25,8 +36,8 @@ module Wayfarer
 
       def supervise
         yield
-      rescue *strategy.renew_on, *Wayfarer.config.network.renew_on => e
-        renew
+      rescue *strategy.renew_on, *Wayfarer.config[:network][:renew_on] => e
+        renew # may raise
       ensure
         # If renewing raises, re-raise the originally caught exception
         # TODO: Not nice this effectively swallows exceptions

data/lib/wayfarer/networking/ferrum.rb

@@ -10,13 +10,10 @@ module Wayfarer
       end
 
       def create
-        ::Ferrum::Browser.new(Wayfarer.config.ferrum.options).tap do |browser|
-          browser.headers.set(Wayfarer.config.network.http_headers)
-        end
+        ::Ferrum::Browser.new(Wayfarer.config.dig(:ferrum, :options))
       end
 
       def destroy(instance)
-        instance.reset
         instance.quit
       end
 

data/lib/wayfarer/networking/follow.rb

@@ -2,17 +2,24 @@
 
 module Wayfarer
   module Networking
-    RedirectsExhaustedError = Class.new(StandardError)
+    class Follow
+      RedirectsExhaustedError = Class.new(StandardError)
 
-    Follow = Struct.new(:context) do
-      extend Forwardable
+      def self.http
+        new(Wayfarer::Networking::Context.new(Wayfarer::Networking::HTTP.new))
+      end
+
+      attr_reader :context
 
-      delegate %i[live renew instance] => :context
+      def initialize(context)
+        @context = context
+      end
 
-      def fetch(url, follow: 3)
-        raise RedirectsExhaustedError if follow.negative?
+      def fetch(url, follow:)
+        raise RedirectsExhaustedError if follow < 0
 
-        case result = context.fetch(url)
+        result = context.fetch(url)
+        case result
         when Result::Success then result.page
         when Result::Redirect then fetch(result.redirect_url, follow: follow - 1)
         end