wayfarer 0.4.7 → 0.4.9

data/lib/wayfarer/cli/route_printer.rb

@@ -2,128 +2,110 @@
 
 module Wayfarer
   class CLI
-    class RoutePrinter < Thor::Shell::Color
-      attr_reader :url, :path_finder, :output
+    # Turns a routing tree into a Hash and prints it.
+    # Used by the `route` CLI subcommand.
+    #
+    # @api private
+    class RoutePrinter
+      # @return [Hash<Symbol, Proc>]
+      class_attribute :serializers,
+                      default: { yaml: ->(hash) { YAML.dump(hash.deep_stringify_keys) },
+                                 json: ->(hash) { JSON.pretty_generate(hash) },
+                                 ruby: ->(hash) { pp(hash) } },
+                      instance_accessor: false,
+                      instance_predicate: false
 
-      INDENT = "   "
-      REGULAR_SEGMENT = "│  "
-      JUNCTION_SEGMENT = "├──"
-      CORNER_SEGMENT = "└──"
+      BATCH = "tmp"
 
-      def self.print(route, url)
-        route.accept(new(url))
+      # Prints a routing tree.
+      #
+      # @param route [Wayfarer::Routing::Route] route to print
+      # @param url [String] URL to match
+      # @param format [String, Symbol] `:json`, `:yaml` or `:ruby`
+      def self.print(route, url, format:)
+        new(route, url, serializers.fetch(format.to_sym)).print
       end
 
-      def initialize(url)
-        @url = url
-        @path_finder = Wayfarer::Routing::PathFinder.new(url)
-        super()
-      end
-
-      def visit(route)
-        route.accept(path_finder) unless route.parent
-        puts format_route_output(route)
-        true
-      end
-
-    private
+      # @param route [Wayfarer::Routing::Route] route to print
+      # @param url [String] URL to match
+      # @param serializer [Proc<Hash=>String>] output serializer
+      def initialize(route, url, serializer)
+        @route = route
+        @serializer = serializer
 
-      def format_route_output(route)
-        [segments(route), route_description(route)].join[3..]
-      end
+        @nodes = {}
+        @root_hash = nil
 
-      def segments(route)
-        [parents(route).map { |parent| parent_segment(parent) }, segment(route)].join
+        task = Wayfarer::Task.new(url, BATCH)
+        task[:uri] = Addressable::URI.parse(url)
+        @path_finder = Wayfarer::Routing::PathFinder.new(
+          task,
+          stop_when_found: false,
+          &method(:call)
+        )
       end
 
-      def parent_segment(parent)
-        trailer?(parent) ? INDENT : REGULAR_SEGMENT
-      end
+      # Processes the routing trees and prints the serialized output.
+      def print
+        route.accept(path_finder)
 
-      def segment(route)
-        trailer?(route) ? CORNER_SEGMENT : JUNCTION_SEGMENT
-      end
+        hash = routing_result(path_finder).merge(root_hash)
 
-      def route_description(route)
-        attrs = [route_arg(route), routing_result(route), route_action(route), route_params(route)].compact
-        text = attrs.any? ? "#{matcher_name(route)}(#{attrs.join(', ')})" : matcher_name(route)
-        set_color(text, *route_colors(route))
+        puts serializer.call(hash)
       end
 
-      def matcher_name(route)
-        case route
-        when Wayfarer::Routing::TargetRoute
-          "Target"
-        when Wayfarer::Routing::RootRoute
-          Wayfarer::Routing::PathFinder.result(route, url).class.name.demodulize
-        else
-          route.matcher.class.name.demodulize
-        end
-      end
+      # Callback method called by `path_finder` with the result of matching
+      # the route.
+      #
+      # @param route [Wayfarer::Routing::Route] the current route
+      # @param result [true, false] routing result
+      # @param path_finder [Wayfarer::Routing::PathFinder] the path finder
+      def call(route, result, path_finder)
+        node = (nodes[route] ||= attributes(route, result, path_finder))
+        parent = route.parent
 
-      def routing_result(route)
-        return if route.is_a?(Wayfarer::Routing::RootRoute)
+        return @root_hash ||= node unless parent
 
-        "match: #{route.matcher.match(url)}"
+        nodes.dig(parent, route_type(parent), :children).append(node)
       end
 
-      def route_action(route)
-        return unless route.is_a?(Wayfarer::Routing::RootRoute)
-
-        result = Wayfarer::Routing::PathFinder.result(route, url)
-        result.action.inspect if result.is_a?(Wayfarer::Routing::Result::Match)
-      end
+    private
 
-      def route_arg(route)
-        return if route.is_a?(Wayfarer::Routing::RootRoute) || route.is_a?(Wayfarer::Routing::TargetRoute)
-
-        matcher = route.matcher
-        matcher_opts = case matcher
-                       when Wayfarer::Routing::Matchers::Host then matcher.host
-                       when Wayfarer::Routing::Matchers::Path then matcher.path
-                       when Wayfarer::Routing::Matchers::Query then matcher.fields
-                       when Wayfarer::Routing::Matchers::Custom then route.action.to_s
-                       when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
-                       when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
-                       end
-        matcher_opts.inspect
-      end
+      attr_reader :route,
+                  :path_finder,
+                  :serializer,
+                  :nodes,
+                  :root_hash
 
-      def route_params(route)
-        params = if route.is_a?(Wayfarer::Routing::RootRoute)
-                   result = Wayfarer::Routing::PathFinder.result(route, url)
-                   result.params if result.is_a?(Wayfarer::Routing::Result::Match)
-                 else
-                   route.matcher.params(url)
-                 end
+      def routing_result(path_finder)
+        return { routed: false } unless path_finder.found?
 
-        "params: #{params.symbolize_keys}" if params&.any?
+        action = path_finder.action
+        { routed: true,
+          params: path_finder.params,
+          action: case action
+                  when Array
+                    { handler: action.first.class.name, action: action.second }
+                  else action
+                  end }
       end
 
-      def parents(route, current = [])
-        return current unless route.parent
-
-        parents(route.parent, [route.parent, *current])
+      def attributes(route, result, path_finder)
+        { route_type(route) => route.to_h.merge!(
+          route_result(route, result, path_finder),
+          children: []
+        ) }
       end
 
-      def trailer?(route)
-        !route.parent || route.parent.children.last == route
-      end
+      def route_result(route, result, path_finder)
+        return {} if route.target?
 
-      def route_colors(route)
-        if path_finder.path.include?(route)
-          %i[green bold]
-        elsif route.matcher.match(url)
-          %i[green]
-        else
-          %i[red]
-        end
+        { match: result,
+          params: path_finder.params_stack.to_h }
       end
 
-      def set_color(string, *colors)
-        return string if ENV.key?("NO_COLOR")
-
-        super(string, *colors)
+      def route_type(route)
+        route.class.name.demodulize.underscore
       end
     end
   end

data/lib/wayfarer/cli.rb

@@ -13,36 +13,16 @@ module Wayfarer
 
     class_option :require, aliases: :r, type: :string, default: nil
 
-    desc "route JOB URL", "Routing result of URL for JOB"
+    desc "route JOB URL", "Routing tree for URL for JOB"
+    option :format, type: :string, enum: %w[yaml json ruby], default: "yaml"
     def route(job, url)
       load_environment
 
       url = parsed_url(url)
-      job = job.classify.constantize
+      job = job.camelize.constantize
       route = job.route
-      route.invoke(url)
 
-      result = Wayfarer::Routing::PathFinder.result(route, url)
-      result_type = result.class.name.demodulize
-
-      say case result
-          when Wayfarer::Routing::Result::Match
-            "#{result_type} => #{result.action.inspect}"
-          else
-            result_type
-          end
-    end
-
-    desc "tree JOB URL", "Visualize JOB's routing tree for URL"
-    def tree(job, url)
-      load_environment
-
-      url = parsed_url(url)
-      job = job.classify.constantize
-      route = job.route
-      route.invoke(url)
-
-      Wayfarer::CLI::RoutePrinter.print(route, url)
+      Wayfarer::CLI::RoutePrinter.print(route, url, format: options.fetch("format"))
     end
 
     desc "perform JOB URL", "Perform JOB with URL"
@@ -52,8 +32,8 @@ module Wayfarer
       load_environment
       mock_redis
 
-      job = job.classify.constantize
-      task = Wayfarer::Task.new(url, options[:batch])
+      job = job.camelize.constantize
+      task = Wayfarer::Task.new(url, options.fetch(:batch))
       job.new(task).perform_now
     end
 
@@ -62,7 +42,7 @@ module Wayfarer
     def enqueue(job, url)
       load_environment
 
-      job = job.classify.constantize
+      job = job.camelize.constantize
       job.crawl(url, batch: options[:batch])
     end
 
@@ -71,21 +51,23 @@ module Wayfarer
     option :batch, type: :string, default: SecureRandom.uuid
     option :min_threads, type: :numeric, default: 1
     option :max_threads, type: :numeric, default: 1
+    option :retain_pool, type: :boolean, default: false
     def execute(job, url)
       load_environment
       mock_redis
 
-      job = job.classify.constantize
+      job = job.camelize.constantize
       job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
                                                                      max_threads: options[:max_threads])
       scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
       executor = scheduler.instance_variable_get(:@async_executor)
 
-      job.crawl(url, batch: options[:batch])
+      job.crawl(url, batch: options.fetch(:batch))
 
       sleep(0.1) while executor.scheduled_task_count > executor.completed_task_count
 
-      Wayfarer::Networking::Pool.instance.free
+      # Used in test suite to avoid pool recreation
+      Wayfarer::Networking::Pool.instance.free unless options.fetch(:retain_pool)
     end
 
   private

data/lib/wayfarer/gc.rb

@@ -6,12 +6,17 @@ module Wayfarer
     RESETTABLES = [Wayfarer::Redis::Barrier, Wayfarer::Redis::Counter].freeze
 
     class << self
-      include Wayfarer::Logging.emit(gc: [:info, "Garbage collecting %<resettable>s"])
+      include Wayfarer::Logging.emit(
+        after_batch: [:debug, "Running `after_batch` callback"],
+        gc: [:debug, "Garbage collecting %<resettable>s"]
+      )
     end
 
   module_function
 
     def run(task)
+      task[:job].run_callbacks(:batch)
+
       RESETTABLES.each do |resettable|
         log(:gc, task, resettable: resettable)
         resettable.new(task).reset!

data/lib/wayfarer/kv.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  # Provides a key-value store via `[]` and `[]=`.
+  #
+  # @api private
+  module KV
+    # @param key [Object] key to fetch
+    # @return [Object, nil] value associated with the key or `nil`
+    def [](key)
+      kv[key]
+    end
+
+    # @param key [Object] key to set
+    # @param value [Object] value to set
+    # @return [Object] value that was set
+    def []=(key, value)
+      kv[key] = value
+    end
+
+  private
+
+    # @return [Hash<Object, Object>]
+    def kv
+      @kv ||= {}
+    end
+  end
+end

data/lib/wayfarer/middleware/base.rb

@@ -12,10 +12,6 @@ module Wayfarer
 
         Module.new
       end
-
-      def lazy
-        Wayfarer::Middleware::Lazy.new(self)
-      end
     end
   end
 end

data/lib/wayfarer/middleware/chain.rb

@@ -2,9 +2,15 @@
 
 module Wayfarer
   module Middleware
-    Chain = Struct.new(:middlewares) do
+    class Chain
       extend Forwardable
 
+      attr_reader :middlewares
+
+      def initialize(middlewares)
+        @middlewares = middlewares
+      end
+
       def self.empty
         new([])
       end

data/lib/wayfarer/middleware/content_type.rb

@@ -16,13 +16,16 @@ module Wayfarer
         end
 
         class_methods do
-          def content_type(*content_types)
-            content_types.each do |content_type|
-              case content_type
-              when String then allowed_content_types[:index][content_type] = true
-              when Regexp then allowed_content_types[:patterns] << content_type
-              end
-            end
+          def content_type(*types)
+            grouped_types = types.group_by(&:class)
+
+            new_index = grouped_types.fetch(String, []).index_with { true }
+            new_patterns = grouped_types.fetch(Regexp, [])
+
+            self.allowed_content_types = {
+              index: allowed_content_types.fetch(:index).merge(new_index),
+              patterns: allowed_content_types.fetch(:patterns).merge(new_patterns)
+            }
           end
         end
       end
@@ -35,19 +38,21 @@ module Wayfarer
 
       def permitted?(task)
         job = task[:controller]
-        content_types = job.class.allowed_content_types
+        rules = job.class.allowed_content_types
 
-        return true if allows_all?(content_types)
+        return true if no_restrictions?(rules)
 
-        content_type = task[:page].mime_type&.to_s || task[:page].headers["content-type"] || (return false)
+        return false unless (content_type = task[:page].mime_type&.to_s || task[:page].content_type)
 
-        content_types[:index].key?(content_type) || content_types[:patterns].any? do |pattern|
-          pattern.match?(content_type)
-        end
+        content_type && matches_rules?(content_type, rules)
+      end
+
+      def matches_rules?(type, rules)
+        rules.fetch(:index).key?(type) || rules.fetch(:patterns).any? { |p| p.match?(type) }
       end
 
-      def allows_all?(content_types)
-        content_types[:index].empty? && content_types[:patterns].empty?
+      def no_restrictions?(rules)
+        rules.values_at(:index, :patterns).all?(&:empty?)
       end
     end
   end

data/lib/wayfarer/middleware/controller.rb

@@ -18,7 +18,7 @@ module Wayfarer
 
       class_methods do
         def use(middleware)
-          chain.push(middleware.lazy)
+          chain.push(middleware.new)
           api(middleware)
         end
 

data/lib/wayfarer/middleware/dedup.rb

@@ -16,18 +16,24 @@ module Wayfarer
 
         if task[:job].executions > 1
           log(:retry, task)
-          return yield if block_given?
+          return (yield if block_given?)
         end
 
         if task[:job] != task[:controller]
           log(:rerouted, task)
-          return yield if block_given?
+          return (yield if block_given?)
         end
 
-        return log(:deduplicated, task) if task[:barrier].check!(task[:normalized_url])
+        return log(:deduplicated, task) if task[:barrier].check!(key(task))
 
         yield if block_given?
       end
+
+    private
+
+      def key(task)
+        Wayfarer.config.dig(:deduplication, :key).call(task)
+      end
     end
   end
 end

data/lib/wayfarer/middleware/dispatch.rb

@@ -5,6 +5,8 @@ module Wayfarer
     class Dispatch
       extend Base
 
+      InvalidTargetError = Class.new(ArgumentError)
+
       def call(task)
         controller = task[:controller]
 
@@ -16,9 +18,12 @@ module Wayfarer
             task[:action] = method
             handler.new.call(task)
           else
-            raise ArgumentError, "invalid action: #{action.inspect}" unless action&.include?(Wayfarer::Handler)
+            unless action&.include?(Wayfarer::Handler)
+              raise InvalidTargetError, "routed to invalid action: #{action.inspect}"
+            end
+
+            task[:action] = nil
 
-            task[:action] = nil # TODO: Test
             action.new.call(task)
           end
         end

data/lib/wayfarer/middleware/normalize.rb

@@ -5,23 +5,15 @@ module Wayfarer
     class Normalize
       extend Base
       include Wayfarer::Logging.emit(
-        invalid: [:info, "Failed to normalize HTTP(S) URL"]
+        invalid: [:info, "Failed to normalize URL"]
       )
 
-      def self.normalize(uri)
-        return uri.to_s unless %w[http https].include?(uri.scheme)
-
-        NormalizeUrl.process(uri)
-      rescue NormalizeUrl::InvalidURIError
-        nil
-      end
-
       def call(task)
-        return (yield if block_given?) if task[:normalized_url]
-
-        return log(:invalid, task) unless (task[:normalized_url] = self.class.normalize(task[:uri]))
+        Wayfarer::URI::Normalization.canonical!(task[:uri])
 
         yield if block_given?
+      rescue Wayfarer::URI::Normalization::InvalidURIError
+        log(:invalid, task)
       end
     end
   end

data/lib/wayfarer/middleware/router.rb

@@ -38,7 +38,7 @@ module Wayfarer
           return (yield if block_given?)
         end
 
-        case result = task[:controller].class.route.invoke(task[:uri])
+        case result = task[:controller].class.route.invoke(task)
         when Routing::Result::Mismatch then return log(:mismatch, task)
         when Routing::Result::Match
           action = result.action

data/lib/wayfarer/middleware/uri_parser.rb

@@ -16,9 +16,10 @@ module Wayfarer
       end
 
       def call(task)
-        # TODO: Test
-        task[:uri] ||= begin
-          Addressable::URI.parse(task.url).normalize
+        return (yield if block_given?) if task[:uri]
+
+        begin
+          task[:uri] = Addressable::URI.parse(task.url)
         rescue Addressable::URI::InvalidURIError => e
           return log(:invalid, task, message: e.message)
         end

data/lib/wayfarer/networking/context.rb

@@ -2,11 +2,22 @@
 
 module Wayfarer
   module Networking
-    Context = Struct.new(:strategy) do
+    class Context
+      attr_reader :strategy
+
+      def initialize(strategy)
+        @strategy = strategy
+      end
+
       def fetch(url)
         supervise { strategy.fetch(instance, url) }
       end
 
+      def navigate(url)
+        fetch(url)
+        live
+      end
+
       def live
         supervise { strategy.live(instance) }
       end
@@ -25,7 +36,7 @@ module Wayfarer
 
       def supervise
         yield
-      rescue *strategy.renew_on, *Wayfarer.config[:network][:renew_on] => e
+      rescue *strategy.class.renew_on, *Wayfarer.config[:network][:renew_on] => e
         renew # may raise
       ensure
         # If renewing raises, re-raise the originally caught exception

data/lib/wayfarer/networking/ferrum.rb

@@ -5,18 +5,15 @@ module Wayfarer
     class Ferrum
       include Strategy
 
-      def renew_on
+      def self.renew_on
         [::Ferrum::DeadBrowserError]
       end
 
       def create
-        ::Ferrum::Browser.new(Wayfarer.config[:ferrum][:options]).tap do |browser|
-          browser.headers.set(Wayfarer.config[:network][:http_headers])
-        end
+        ::Ferrum::Browser.new(Wayfarer.config.dig(:ferrum, :options))
       end
 
       def destroy(instance)
-        instance.reset
         instance.quit
       end
 

data/lib/wayfarer/networking/follow.rb

@@ -18,7 +18,8 @@ module Wayfarer
       def fetch(url, follow:)
         raise RedirectsExhaustedError if follow < 0
 
-        case result = context.fetch(url)
+        result = context.fetch(url)
+        case result
         when Result::Success then result.page
         when Result::Redirect then fetch(result.redirect_url, follow: follow - 1)
         end

data/lib/wayfarer/networking/pool.rb

@@ -5,30 +5,27 @@ module Wayfarer
     class Pool
       include Singleton
 
-      class_attribute :registry,
-                      default: { http: HTTP,
-                                 ferrum: Ferrum,
-                                 selenium: Selenium,
-                                 capybara: Capybara },
+      class_attribute :finalizer,
+                      default: ->(pool) { pool.shutdown(&:renew) }.freeze,
                       instance_accessor: false,
                       instance_predicate: false
 
+      attr_reader :pool
+
       def initialize
-        @pool = ConnectionPool.new(size: Wayfarer.config[:network][:pool_size],
-                                   timeout: Wayfarer.config[:network][:pool_timeout],
-                                   &method(:context))
+        @pool = ConnectionPool.new(**Wayfarer.config.dig(:network, :pool), &method(:context))
 
         at_exit { free }
       end
 
-      def with(&block)
-        @pool.with(&block)
+      def with(&)
+        @pool.with(&)
       rescue ConnectionPool::TimeoutError => e
         raise Wayfarer::UserAgentTimeoutError, e
       end
 
       def free
-        @pool.shutdown(&:renew)
+        self.class.finalizer.call(@pool)
       end
 
     private
@@ -38,7 +35,9 @@ module Wayfarer
       end
 
       def strategy
-        self.class.registry[Wayfarer.config[:network][:agent]].new
+        Wayfarer.config.dig(:network, :agent).then do |agent|
+          Wayfarer.config.dig(:network, :agents).fetch(agent).new
+        end
       end
     end
   end