wayfarer 0.4.5 → 0.4.7

data/lib/wayfarer/callbacks.rb

@@ -2,69 +2,43 @@
 
 module Wayfarer
   module Callbacks
-    TERMINATOR = ->(_target, result) { result.call == false }
-    OPTIONS = { terminator: TERMINATOR, skip_after_callbacks_if_terminated: true }.freeze
+    extend ActiveSupport::Concern
 
-    ConditionalCallback = Struct.new(:job, :filters) do
-      def run(method, &block)
-        return if only && !applies?(only)
-        return if except && applies?(except)
+    included do
+      include ActiveSupport::Callbacks
 
-        return job.send(method) if method
+      define_callbacks :fetch, skip_after_callbacks_if_terminated: true
+      define_callbacks :action, skip_after_callbacks_if_terminated: true
+      define_callbacks :batch
+    end
 
-        job.instance_eval(&block)
+    class_methods do
+      def before_fetch(*filters, &block)
+        set_callback(:fetch, :before, *filters, &block)
       end
 
-    private
-
-      def applies?(condition)
-        case condition
-        when Symbol then condition == action
-        when Enumerable then condition&.include?(action)
-        end
+      def around_fetch(*filters, &block)
+        set_callback(:fetch, :around, *filters, &block)
       end
 
-      def only
-        filters[:only]
+      def after_fetch(*filters, &block)
+        set_callback(:fetch, :after, *filters, &block)
       end
 
-      def except
-        filters[:except]
+      def before_action(*filters, &block)
+        set_callback(:action, :before, *filters, &block)
       end
 
-      def action
-        task.metadata.action
+      def around_action(*filters, &block)
+        set_callback(:action, :around, *filters, &block)
       end
 
-      def task
-        job.task
+      def after_action(*filters, &block)
+        set_callback(:action, :after, *filters, &block)
       end
-    end
-
-    def self.included(base)
-      base.include(ActiveSupport::Callbacks)
-      base.extend(ClassMethods)
-
-      base.class_eval do
-        define_callbacks(:fetch, OPTIONS)
-        define_callbacks(:action, OPTIONS)
-        define_callbacks(:batch, OPTIONS)
-
-        define(:fetch, :before)
-        define(:action, :before)
-        define(:batch, :after)
-      end
-    end
-
-    module ClassMethods
-    private
 
-      def define(name, stage)
-        define_singleton_method([stage, name].join("_")) do |method = nil, **filters, &block|
-          set_callback(name, stage, **filters) do |job|
-            ConditionalCallback.new(job, filters).run(method, &block)
-          end
-        end
+      def after_batch(*filters, &block)
+        set_callback(:batch, :after, *filters, &block)
       end
     end
   end

data/lib/wayfarer/cli/route_printer.rb

@@ -1,16 +1,14 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  module CLI
+  class CLI
     class RoutePrinter < Thor::Shell::Color
-      attr_reader :url,
-                  :path_finder,
-                  :output
+      attr_reader :url, :path_finder, :output
 
-      INDENT           = "   "
-      REGULAR_SEGMENT  = "│  "
+      INDENT = "   "
+      REGULAR_SEGMENT = "│  "
       JUNCTION_SEGMENT = "├──"
-      CORNER_SEGMENT   = "└──"
+      CORNER_SEGMENT = "└──"
 
       def self.print(route, url)
         route.accept(new(url))
@@ -24,81 +22,83 @@ module Wayfarer
 
       def visit(route)
         route.accept(path_finder) unless route.parent
-        return true if route.is_a?(Wayfarer::Routing::RootRoute)
-
-        puts [segments(route), label(route)].join("")[3..]
+        puts format_route_output(route)
         true
       end
 
+    private
+
+      def format_route_output(route)
+        [segments(route), route_description(route)].join[3..]
+      end
+
       def segments(route)
-        current = segment(route)
-        parents = parents(route).map { |parent| parent_segment(parent) }
-        [parents, current].join
+        [parents(route).map { |parent| parent_segment(parent) }, segment(route)].join
       end
 
       def parent_segment(parent)
-        if trailer?(parent)
-          INDENT
-        else
-          REGULAR_SEGMENT
-        end
+        trailer?(parent) ? INDENT : REGULAR_SEGMENT
       end
 
       def segment(route)
-        if trailer?(route)
-          CORNER_SEGMENT
-        else
-          JUNCTION_SEGMENT
-        end
+        trailer?(route) ? CORNER_SEGMENT : JUNCTION_SEGMENT
       end
 
-      def label(route)
-        [highlight_matcher(route, matcher_label(route)),
-         highlight_options(route, options(route)),
-         highlight_options(route, params(route))].compact.join(" ")
+      def route_description(route)
+        attrs = [route_arg(route), routing_result(route), route_action(route), route_params(route)].compact
+        text = attrs.any? ? "#{matcher_name(route)}(#{attrs.join(', ')})" : matcher_name(route)
+        set_color(text, *route_colors(route))
       end
 
-      def highlight_matcher(route, string)
-        if path_finder.path.include?(route)
-          set_color(string, :green, :bold)
-        elsif route.matcher.match(url)
-          set_color(string, :green)
+      def matcher_name(route)
+        case route
+        when Wayfarer::Routing::TargetRoute
+          "Target"
+        when Wayfarer::Routing::RootRoute
+          Wayfarer::Routing::PathFinder.result(route, url).class.name.demodulize
         else
-          set_color(string, :red)
+          route.matcher.class.name.demodulize
         end
       end
 
-      def highlight_options(route, string)
-        return string unless path_finder.path.include?(route)
+      def routing_result(route)
+        return if route.is_a?(Wayfarer::Routing::RootRoute)
 
-        set_color(string, :green, :bold)
+        "match: #{route.matcher.match(url)}"
       end
 
-      def matcher_label(route)
-        return "Target" if route.is_a?(Wayfarer::Routing::TargetRoute)
+      def route_action(route)
+        return unless route.is_a?(Wayfarer::Routing::RootRoute)
 
-        route.matcher.class.name.demodulize
+        result = Wayfarer::Routing::PathFinder.result(route, url)
+        result.action.inspect if result.is_a?(Wayfarer::Routing::Result::Match)
       end
 
-      def options(route)
-        return "" if route.is_a?(Wayfarer::Routing::RootRoute)
-
-        case (matcher = route.matcher)
-        when Wayfarer::Routing::Matchers::Host then matcher.host
-        when Wayfarer::Routing::Matchers::Path then matcher.path
-        when Wayfarer::Routing::Matchers::Query then matcher.fields
-        when Wayfarer::Routing::Matchers::Custom then "##{route.action}"
-        when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
-        when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
-        end
+      def route_arg(route)
+        return if route.is_a?(Wayfarer::Routing::RootRoute) || route.is_a?(Wayfarer::Routing::TargetRoute)
+
+        matcher = route.matcher
+        matcher_opts = case matcher
+                       when Wayfarer::Routing::Matchers::Host then matcher.host
+                       when Wayfarer::Routing::Matchers::Path then matcher.path
+                       when Wayfarer::Routing::Matchers::Query then matcher.fields
+                       when Wayfarer::Routing::Matchers::Custom then route.action.to_s
+                       when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
+                       when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
+                       end
+        matcher_opts.inspect
       end
 
-      def params(route)
-        params = route.matcher.params(url)
-        "=> #{params.symbolize_keys}" if params.any?
-      end
+      def route_params(route)
+        params = if route.is_a?(Wayfarer::Routing::RootRoute)
+                   result = Wayfarer::Routing::PathFinder.result(route, url)
+                   result.params if result.is_a?(Wayfarer::Routing::Result::Match)
+                 else
+                   route.matcher.params(url)
+                 end
 
-    private
+        "params: #{params.symbolize_keys}" if params&.any?
+      end
 
       def parents(route, current = [])
         return current unless route.parent
@@ -107,9 +107,23 @@ module Wayfarer
       end
 
       def trailer?(route)
-        return true unless route.parent
+        !route.parent || route.parent.children.last == route
+      end
+
+      def route_colors(route)
+        if path_finder.path.include?(route)
+          %i[green bold]
+        elsif route.matcher.match(url)
+          %i[green]
+        else
+          %i[red]
+        end
+      end
+
+      def set_color(string, *colors)
+        return string if ENV.key?("NO_COLOR")
 
-        route.parent.children.last == route
+        super(string, *colors)
       end
     end
   end

data/lib/wayfarer/cli.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    desc "version", "Print version"
+    def version
+      say Wayfarer::VERSION::STRING
+    end
+
+    class_option :require, aliases: :r, type: :string, default: nil
+
+    desc "route JOB URL", "Routing result of URL for JOB"
+    def route(job, url)
+      load_environment
+
+      url = parsed_url(url)
+      job = job.classify.constantize
+      route = job.route
+      route.invoke(url)
+
+      result = Wayfarer::Routing::PathFinder.result(route, url)
+      result_type = result.class.name.demodulize
+
+      say case result
+          when Wayfarer::Routing::Result::Match
+            "#{result_type} => #{result.action.inspect}"
+          else
+            result_type
+          end
+    end
+
+    desc "tree JOB URL", "Visualize JOB's routing tree for URL"
+    def tree(job, url)
+      load_environment
+
+      url = parsed_url(url)
+      job = job.classify.constantize
+      route = job.route
+      route.invoke(url)
+
+      Wayfarer::CLI::RoutePrinter.print(route, url)
+    end
+
+    desc "perform JOB URL", "Perform JOB with URL"
+    option :mock_redis, type: :boolean
+    option :batch, type: :string, default: SecureRandom.uuid
+    def perform(job, url)
+      load_environment
+      mock_redis
+
+      job = job.classify.constantize
+      task = Wayfarer::Task.new(url, options[:batch])
+      job.new(task).perform_now
+    end
+
+    desc "enqueue JOB URL", "Enqueue JOB with URL"
+    option :batch, type: :string, default: SecureRandom.uuid
+    def enqueue(job, url)
+      load_environment
+
+      job = job.classify.constantize
+      job.crawl(url, batch: options[:batch])
+    end
+
+    desc "execute JOB URL", "Execute JOB with async adapter starting from URL"
+    option :mock_redis, type: :boolean
+    option :batch, type: :string, default: SecureRandom.uuid
+    option :min_threads, type: :numeric, default: 1
+    option :max_threads, type: :numeric, default: 1
+    def execute(job, url)
+      load_environment
+      mock_redis
+
+      job = job.classify.constantize
+      job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
+                                                                     max_threads: options[:max_threads])
+      scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
+      executor = scheduler.instance_variable_get(:@async_executor)
+
+      job.crawl(url, batch: options[:batch])
+
+      sleep(0.1) while executor.scheduled_task_count > executor.completed_task_count
+
+      Wayfarer::Networking::Pool.instance.free
+    end
+
+  private
+
+    def mock_redis
+      Wayfarer.config[:redis][:factory] = ->(_) { MockRedis.new } if options[:mock_redis]
+    end
+
+    def parsed_url(url)
+      Addressable::URI.parse(url).normalize
+    end
+
+    def load_environment(require_path = options[:require])
+      require File.join(Dir.pwd, require_path) if require_path
+
+      load_rails
+    end
+
+    def load_rails
+      begin
+        require "rails/app_loader"
+      rescue LoadError
+        return
+      end
+
+      return unless Rails::AppLoader.find_executable
+
+      require File.expand_path("config/application", Dir.pwd)
+      require File.expand_path("config/boot", Dir.pwd)
+      require File.expand_path("config/environment", Dir.pwd)
+    end
+  end
+end

data/lib/wayfarer/gc.rb

@@ -1,14 +1,21 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  GC = Struct.new(:task) do
-    def run
-      return unless task.counter.decrement <= 0
+  # TODO: Add logging
+  module GC
+    RESETTABLES = [Wayfarer::Redis::Barrier, Wayfarer::Redis::Counter].freeze
 
-      task.metadata.job.run_callbacks(:batch)
+    class << self
+      include Wayfarer::Logging.emit(gc: [:info, "Garbage collecting %<resettable>s"])
+    end
+
+  module_function
 
-      task.barrier.reset!
-      task.counter.reset!
+    def run(task)
+      RESETTABLES.each do |resettable|
+        log(:gc, task, resettable: resettable)
+        resettable.new(task).reset!
+      end
     end
   end
 end

data/lib/wayfarer/handler.rb

@@ -1,15 +1,23 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  class Handler
-    include Wayfarer::Middleware::Controller
+  module Handler
+    extend ActiveSupport::Concern
 
-    api Wayfarer::Middleware::Fetch
-    api Wayfarer::Middleware::Stage
+    included do
+      include Wayfarer::Middleware::Controller
 
-    use Wayfarer::Middleware::Router
-    use Wayfarer::Middleware::Dispatch
+      use Wayfarer::Middleware::ContentType
+      use Wayfarer::Middleware::Router
+      use Wayfarer::Middleware::Dispatch
 
-    singleton_class.undef_method :after_batch
+      api Wayfarer::Middleware::UserAgent
+      api Wayfarer::Middleware::Stage
+
+      singleton_class.undef_method :before_fetch
+      singleton_class.undef_method :around_fetch
+      singleton_class.undef_method :after_fetch
+      singleton_class.undef_method :after_batch
+    end
   end
 end

data/lib/wayfarer/logging.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Logging
+    mattr_accessor :logger, default: ActiveSupport::Logger.new($stdout)
+
+    def self.emit(...)
+      Emitter.new(...)
+    end
+
+    class Emitter < Module
+      def initialize(messages)
+        @messages = messages
+
+        super()
+      end
+
+      def included(base)
+        messages = @messages
+
+        base.class_eval do
+          define_method(:log) do |key, task, **args|
+            level, msg = messages[key] || raise(ArgumentError, "No log message for #{key.inspect}")
+            severity = ActiveSupport::Logger::Severity.const_get(level.upcase)
+
+            ActiveSupport::TaggedLogging
+              .new(Logging.logger)
+              .tagged(task.batch, task.url, task[:controller]&.class&.name) do |logger|
+              logger.add(severity, msg % args)
+            end
+          end
+        end
+      end
+    end
+
+    private_constant :Emitter
+  end
+end

data/lib/wayfarer/middleware/base.rb

@@ -3,6 +3,8 @@
 module Wayfarer
   module Middleware
     module Base
+      extend ActiveSupport::Concern
+
       API_MODULE = :API
 
       def api

data/lib/wayfarer/middleware/batch_completion.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class BatchCompletion
+      extend Base
+
+      def call(task)
+        # Comparing to the initial state of `exception_executions` allows
+        # us to determine if an exception occurred when the job was performed,
+        # since the `perform.active_job` event is emitted for both successful
+        # and raising jobs.
+        task[:initial_exception_executions] ||= task[:job].exception_executions.clone
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/content_type.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class ContentType
+      extend Base
+
+      module API
+        extend ActiveSupport::Concern
+
+        included do
+          class_attribute :allowed_content_types,
+                          default: { index: {}, patterns: Set.new },
+                          instance_accessor: false,
+                          instance_predicate: false
+        end
+
+        class_methods do
+          def content_type(*content_types)
+            content_types.each do |content_type|
+              case content_type
+              when String then allowed_content_types[:index][content_type] = true
+              when Regexp then allowed_content_types[:patterns] << content_type
+              end
+            end
+          end
+        end
+      end
+
+      def call(task)
+        yield if block_given? && permitted?(task)
+      end
+
+    private
+
+      def permitted?(task)
+        job = task[:controller]
+        content_types = job.class.allowed_content_types
+
+        return true if allows_all?(content_types)
+
+        content_type = task[:page].mime_type&.to_s || task[:page].headers["content-type"] || (return false)
+
+        content_types[:index].key?(content_type) || content_types[:patterns].any? do |pattern|
+          pattern.match?(content_type)
+        end
+      end
+
+      def allows_all?(content_types)
+        content_types[:index].empty? && content_types[:patterns].empty?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/controller.rb

@@ -3,16 +3,20 @@
 module Wayfarer
   module Middleware
     module Controller
-      def self.included(base)
-        base.cattr_accessor :chain, default: Chain.empty
-        base.attr_accessor :task
+      extend ActiveSupport::Concern
 
-        base.extend(ClassMethods)
-        base.include(InstanceMethods)
-        base.include(Wayfarer::Callbacks)
+      included do
+        class_attribute :chain,
+                        default: Chain.empty,
+                        instance_accessor: false,
+                        instance_predicate: false
+
+        attr_accessor :task
+
+        include Wayfarer::Callbacks
       end
 
-      module ClassMethods
+      class_methods do
         def use(middleware)
           chain.push(middleware.lazy)
           api(middleware)
@@ -23,17 +27,17 @@ module Wayfarer
         end
       end
 
-      module InstanceMethods
-        def call(task)
-          self.task = task
+      def call(task)
+        self.task = task
 
-          task.metadata.job ||= self
-          task.metadata.controller = self
+        task[:job] ||= self
+        task[:controller] = self
 
-          self.class.chain.call(task) do
-            yield if block_given?
-          end
+        self.class.chain.call(task) do
+          yield if block_given?
         end
+
+        task[:return_value]
       end
     end
   end

data/lib/wayfarer/middleware/dedup.rb

@@ -5,25 +5,28 @@ module Wayfarer
     class Dedup
       extend Base
 
+      include Wayfarer::Logging.emit(
+        deduplicated: [:info, "Deduplicated URL"],
+        retry: [:debug, "Not deduplicating retry"],
+        rerouted: [:debug, "Not deduplicating rerouted task"]
+      )
+
       def call(task)
-        # Was task routed by a previous controller already?
-        return yield if task.metadata.action
+        task[:barrier] ||= Wayfarer::Redis::Barrier.new(task)
 
-        return if task.barrier.seen?(task.url)
+        if task[:job].executions > 1
+          log(:retry, task)
+          return yield if block_given?
+        end
 
-        begin
-          yield if block_given?
-        rescue StandardError => e
-          task.barrier.unsee(task.url)
-          raise e
+        if task[:job] != task[:controller]
+          log(:rerouted, task)
+          return yield if block_given?
         end
 
-        staged_urls = task.metadata.staged_urls
-        return if staged_urls.none?
+        return log(:deduplicated, task) if task[:barrier].check!(task[:normalized_url])
 
-        inclusion = task.barrier.peek(staged_urls.to_a)
-        unseen = staged_urls.zip(inclusion).reject { |_, seen| seen }.map(&:first)
-        task.metadata.staged_urls = SortedSet.new(unseen)
+        yield if block_given?
       end
     end
   end