wayfarer 0.4.6 → 0.4.8

data/lib/wayfarer/base.rb

@@ -1,60 +1,139 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  class Base < ActiveJob::Base
-    include Wayfarer::Middleware::Controller
-
-    use Wayfarer::Middleware::Stage
-    use Wayfarer::Middleware::Dedup
-    use Wayfarer::Middleware::Normalize
-    use Wayfarer::Middleware::Router
-    use Wayfarer::Middleware::Fetch
-    use Wayfarer::Middleware::Dispatch
-
-    ErrorHandler = lambda do |&block|
-      lambda do |job, error|
-        task = job.arguments.first
-        task.barrier.seen?(task.url)
-        task.gc.run
-        block.call(job, error)
-      end
-    end
+  # @!attribute [r] task
+  #   @return [Wayfarer::Task] the current task
+  # @!attribute [r] uri
+  #   @return [Addressable::URI] parsed task URL
+  # @!attribute [r] user_agent
+  #   @return [Object] the user agent used to retrieve the page
+  # @!attribute [r] action
+  #   @return [Symbol, Handler] action that the task URL was routed to
+  # @!attribute [r] params
+  #   @return [HashWithIndifferentAccess] path parameters collected from routes
+  module Base
+    extend ActiveSupport::Concern
 
-    after_enqueue do |job|
-      task = job.arguments.first
-      task.counter.increment
-    end
+    # @!method stage(urls)
+    # Adds URLs to an internal staging set so that they get enqueued
+    # eventually, once the job executed successfully.
+    # @overload stage(urls)
+    #   @param urls [Array<String>] URLs to add to the staging set.
+    # @overload stage(url)
+    #   @param url [String] URL to add to the staging set.
 
-    after_perform do |job|
-      task = job.arguments.first
-      task.gc.run
-    end
+    # @!method fetch(url, follow: 3)
+    # @param url [String] URL to fetch using plain HTTP(S).
+    # @param follow [Fixnum] Number of redirects to follow.
+    # Retrieves the given URL to a {Page}.
 
-    rescue_from(StandardError) do
-      task = arguments.first
-      task.gc.run
-    end
+    # @!method page(live: false)
+    # @param url [live] whether to retrieve a new {Page}.
+    # @return [Wayfarer::Page]
+    # The most recently retrieved page or a new page for the current task URL if
+    # the `live` keyword is passed.
 
-    def self.retry_on(*argv, &block)
-      super(*argv, &ErrorHandler.call(&block))
-    end
+    # @!scope class
 
-    def self.discard_on(*argv, &block)
-      super(*argv, &ErrorHandler.call(&block))
-    end
+    # @!attribute [r] route
+    # @return [Wayfarer::Routing::DSL]
+    # The job's {Wayfarer::Routing::DSL} that maps URLs to instance methods
+    # or to a {Handler}.
+    # @example Append a host route
+    #   route.host "examplxe.com", to: :index
 
-    def self.crawl(url, batch: SecureRandom.uuid)
-      Task.new(url, batch).tap do |task|
-        perform_later(task)
-      end
-    end
+    # @!method content_types(*content_types)
+    # @param content_types [*Array<String, Regexp>] Content-Types to whitelist
+    # Whitelists Content-Types. Once at least one Content-Type is set, only
+    # those Content-Types will be processed.
+
+    # @!group Callbacks
+
+    # @!method before_fetch
+    # @overload before_fetch(callback)
+    #   @param callback [Symbol] Instance method to call
+    # @overload before_fetch(&block)
+    #   @yield [Wayfarer::Task]
+    # Registers a callback that is called before the page is fetched.
+    # If a symbol is passed, an instance method with the same name will be
+    # called.
+    # @example Accessing the user agent in {#before_fetch}
+    #   before_fetch do |task|
+    #     user_agent # => the user agent that will fetch the page
+    #   end
 
-    def retry_job(...)
-      super(...) # increments the counter by re-enqueuing the job
-      task = arguments.first
-      task.counter.decrement
+    # @!method around_fetch
+    # @overload around_fetch(callback)
+    #   @param callback [Symbol] Instance method to call
+    # @overload around_fetch(&block)
+    #   @yield [Wayfarer::Task]
+    # Registers a callback that is called around the page getting fetched.
+    # If a symbol is passed, an instance method with the same name will be
+    # called.
+
+    # @!method after_fetch
+    # @overload after_fetch(callback)
+    #   @param callback [Symbol] Instance method to call
+    # @overload after_fetch(&block)
+    #   @yield [Wayfarer::Task]
+    # Registers a callback that is called after the page was fetched.
+    # If a symbol is passed, an instance method with the same name will be
+    # called.
+
+    # @!method before_perform
+    # @overload before_perform(callback)
+    #   @param callback [Symbol] Instance method to call
+    # @overload before_perform(&block)
+    #   @yield [Wayfarer::Task]
+    # Registers a callback that is called before the task is performed.
+    # If a symbol is passed, an instance method with the same name will be
+    # called.
+
+    # @!method around_perform
+    # @overload around_perform(callback)
+    #   @param callback [Symbol] Instance method to call
+    # @overload around_perform(&block)
+    #   @yield [Wayfarer::Task]
+    # Registers a callback that is called around the task getting performed.
+    # If a symbol is passed, an instance method with the same name will be
+    # called.
+
+    # @!method after_perform
+    # @overload after_perform(callback)
+    #   @param callback [Symbol] Instance method to call
+    # @overload after_perform(&block)
+    #   @yield [Wayfarer::Task]
+    # Registers a callback that is called after the task was performed.
+    # If a symbol is passed, an instance method with the same name will be
+    # called.
+
+    # @!endgroup
+
+    included do
+      include Wayfarer::Middleware::Controller
+
+      # Implement ActiveJob's #perform by calling into our own middleware
+      # chain included from {Controller}
+      alias_method :perform, :call
+
+      use Wayfarer::Middleware::Redis
+      use Wayfarer::Middleware::UriParser
+      use Wayfarer::Middleware::Normalize
+      use Wayfarer::Middleware::Dedup
+      use Wayfarer::Middleware::BatchCompletion
+      use Wayfarer::Middleware::Stage
+      use Wayfarer::Middleware::Router
+      use Wayfarer::Middleware::UserAgent
+      use Wayfarer::Middleware::ContentType
+      use Wayfarer::Middleware::Dispatch
     end
 
-    alias perform call
+    class_methods do
+      def crawl(url, batch: SecureRandom.uuid)
+        Task.new(url, batch).tap do |task|
+          perform_later(task)
+        end
+      end
+    end
   end
 end

data/lib/wayfarer/batch_completion.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  # BatchCompletion tracks the completion of a batch of jobs.
+  # It does so by incrementing and decrementing a counter in Redis.
+  #
+  # The counter is incremented when a job is first enqueued and decremented when
+  # a job is performed. If a job is retried, the counter is not incremented.
+  # When a job succeeds or fails and thereby exceeds its retry count, the counter
+  # is decremented.
+  #
+  # When the counter reaches zero, garbage collection deletes the Redis keys
+  # associated with the batch.
+  module BatchCompletion
+  module_function
+
+    EVENTS = %w[enqueue.active_job perform.active_job retry_stopped.active_job].freeze
+
+    def subscribe!
+      EVENTS.each { |event| ActiveSupport::Notifications.subscribe(event, self) }
+    end
+
+    def call(name, _, _, _, data)
+      return unless (job = data[:job]).is_a?(Wayfarer::Base)
+
+      task = job.arguments.first
+
+      # In the case of `enqueue.active_job` middleware hasn't executed yet
+      task[:redis_pool] ||= Wayfarer::Redis::Pool.instance
+
+      handle(name, job, task)
+    end
+
+    def handle(name, job, task)
+      counter = Wayfarer::Redis::Counter.new(task)
+
+      case name
+      when "enqueue.active_job" then counter.increment unless retry?(job)
+      when "perform.active_job" then succeed!(task, counter) if succeeded?(job, task)
+      when "retry_stopped.active_job" then fail!(counter)
+      end
+    end
+
+    def succeed!(task, counter)
+      Wayfarer::GC.run(task) if counter.decrement == 0
+    end
+
+    def fail!(counter)
+      counter.decrement
+    end
+
+    def retry?(job)
+      job.executions > 0
+    end
+
+    def succeeded?(job, task)
+      job.exception_executions == task[:initial_exception_executions]
+    end
+  end
+end

data/lib/wayfarer/callbacks.rb

@@ -2,69 +2,43 @@
 
 module Wayfarer
   module Callbacks
-    TERMINATOR = ->(_target, result) { result.call == false }
-    OPTIONS = { terminator: TERMINATOR, skip_after_callbacks_if_terminated: true }.freeze
+    extend ActiveSupport::Concern
 
-    ConditionalCallback = Struct.new(:job, :filters) do
-      def run(method, &block)
-        return if only && !applies?(only)
-        return if except && applies?(except)
+    included do
+      include ActiveSupport::Callbacks
 
-        return job.send(method) if method
+      define_callbacks :fetch, skip_after_callbacks_if_terminated: true
+      define_callbacks :action, skip_after_callbacks_if_terminated: true
+      define_callbacks :batch
+    end
 
-        job.instance_eval(&block)
+    class_methods do
+      def before_fetch(...)
+        set_callback(:fetch, :before, ...)
       end
 
-    private
-
-      def applies?(condition)
-        case condition
-        when Symbol then condition == action
-        when Enumerable then condition&.include?(action)
-        end
+      def around_fetch(...)
+        set_callback(:fetch, :around, ...)
       end
 
-      def only
-        filters[:only]
+      def after_fetch(...)
+        set_callback(:fetch, :after, ...)
       end
 
-      def except
-        filters[:except]
+      def before_action(...)
+        set_callback(:action, :before, ...)
       end
 
-      def action
-        task.metadata.action
+      def around_action(...)
+        set_callback(:action, :around, ...)
       end
 
-      def task
-        job.task
+      def after_action(...)
+        set_callback(:action, :after, ...)
       end
-    end
-
-    def self.included(base)
-      base.include(ActiveSupport::Callbacks)
-      base.extend(ClassMethods)
-
-      base.class_eval do
-        define_callbacks(:fetch, OPTIONS)
-        define_callbacks(:action, OPTIONS)
-        define_callbacks(:batch, OPTIONS)
-
-        define(:fetch, :before)
-        define(:action, :before)
-        define(:batch, :after)
-      end
-    end
-
-    module ClassMethods
-    private
 
-      def define(name, stage)
-        define_singleton_method([stage, name].join("_")) do |method = nil, **filters, &block|
-          set_callback(name, stage, **filters) do |job|
-            ConditionalCallback.new(job, filters).run(method, &block)
-          end
-        end
+      def after_batch(...)
+        set_callback(:batch, :after, ...)
       end
     end
   end

data/lib/wayfarer/cli/route_printer.rb

@@ -1,115 +1,111 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  module CLI
-    class RoutePrinter < Thor::Shell::Color
-      attr_reader :url,
-                  :path_finder,
-                  :output
-
-      INDENT           = "   "
-      REGULAR_SEGMENT  = "│  "
-      JUNCTION_SEGMENT = "├──"
-      CORNER_SEGMENT   = "└──"
-
-      def self.print(route, url)
-        route.accept(new(url))
+  class CLI
+    # Turns a routing tree into a Hash and prints it.
+    # Used by the `route` CLI subcommand.
+    #
+    # @api private
+    class RoutePrinter
+      # @return [Hash<Symbol, Proc>]
+      class_attribute :serializers,
+                      default: { yaml: ->(hash) { YAML.dump(hash.deep_stringify_keys) },
+                                 json: ->(hash) { JSON.pretty_generate(hash) },
+                                 ruby: ->(hash) { pp(hash) } },
+                      instance_accessor: false,
+                      instance_predicate: false
+
+      BATCH = "tmp"
+
+      # Prints a routing tree.
+      #
+      # @param route [Wayfarer::Routing::Route] route to print
+      # @param url [String] URL to match
+      # @param format [String, Symbol] `:json`, `:yaml` or `:ruby`
+      def self.print(route, url, format:)
+        new(route, url, serializers.fetch(format.to_sym)).print
       end
 
-      def initialize(url)
-        @url = url
-        @path_finder = Wayfarer::Routing::PathFinder.new(url)
-        super()
+      # @param route [Wayfarer::Routing::Route] route to print
+      # @param url [String] URL to match
+      # @param serializer [Proc<Hash=>String>] output serializer
+      def initialize(route, url, serializer)
+        @route = route
+        @serializer = serializer
+
+        @nodes = {}
+        @root_hash = nil
+
+        task = Wayfarer::Task.new(url, BATCH)
+        task[:uri] = Addressable::URI.parse(url)
+        @path_finder = Wayfarer::Routing::PathFinder.new(
+          task,
+          stop_when_found: false,
+          &method(:call)
+        )
       end
 
-      def visit(route)
-        route.accept(path_finder) unless route.parent
-        return true if route.is_a?(Wayfarer::Routing::RootRoute)
-
-        puts [segments(route), label(route)].join("")[3..]
-        true
-      end
-
-      def segments(route)
-        current = segment(route)
-        parents = parents(route).map { |parent| parent_segment(parent) }
-        [parents, current].join
-      end
-
-      def parent_segment(parent)
-        if trailer?(parent)
-          INDENT
-        else
-          REGULAR_SEGMENT
-        end
-      end
+      # Processes the routing trees and prints the serialized output.
+      def print
+        route.accept(path_finder)
 
-      def segment(route)
-        if trailer?(route)
-          CORNER_SEGMENT
-        else
-          JUNCTION_SEGMENT
-        end
-      end
+        hash = routing_result(path_finder).merge(root_hash)
 
-      def label(route)
-        [highlight_matcher(route, matcher_label(route)),
-         highlight_options(route, options(route)),
-         highlight_options(route, params(route))].compact.join(" ")
+        puts serializer.call(hash)
       end
 
-      def highlight_matcher(route, string)
-        if path_finder.path.include?(route)
-          set_color(string, :green, :bold)
-        elsif route.matcher.match(url)
-          set_color(string, :green)
-        else
-          set_color(string, :red)
-        end
-      end
+      # Callback method called by `path_finder` with the result of matching
+      # the route.
+      #
+      # @param route [Wayfarer::Routing::Route] the current route
+      # @param result [true, false] routing result
+      # @param path_finder [Wayfarer::Routing::PathFinder] the path finder
+      def call(route, result, path_finder)
+        node = (nodes[route] ||= attributes(route, result, path_finder))
+        parent = route.parent
 
-      def highlight_options(route, string)
-        return string unless path_finder.path.include?(route)
+        return @root_hash ||= node unless parent
 
-        set_color(string, :green, :bold)
+        nodes.dig(parent, route_type(parent), :children).append(node)
       end
 
-      def matcher_label(route)
-        return "Target" if route.is_a?(Wayfarer::Routing::TargetRoute)
-
-        route.matcher.class.name.demodulize
-      end
+    private
 
-      def options(route)
-        return "" if route.is_a?(Wayfarer::Routing::RootRoute)
-
-        case (matcher = route.matcher)
-        when Wayfarer::Routing::Matchers::Host then matcher.host
-        when Wayfarer::Routing::Matchers::Path then matcher.path
-        when Wayfarer::Routing::Matchers::Query then matcher.fields
-        when Wayfarer::Routing::Matchers::Custom then "##{route.action}"
-        when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
-        when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
-        end
+      attr_reader :route,
+                  :path_finder,
+                  :serializer,
+                  :nodes,
+                  :root_hash
+
+      def routing_result(path_finder)
+        return { routed: false } unless path_finder.found?
+
+        action = path_finder.action
+        { routed: true,
+          params: path_finder.params,
+          action: case action
+                  when Array
+                    { handler: action.first.class.name, action: action.second }
+                  else action
+                  end }
       end
 
-      def params(route)
-        params = route.matcher.params(url)
-        "=> #{params.symbolize_keys}" if params.any?
+      def attributes(route, result, path_finder)
+        { route_type(route) => route.to_h.merge!(
+          route_result(route, result, path_finder),
+          children: []
+        ) }
       end
 
-    private
-
-      def parents(route, current = [])
-        return current unless route.parent
+      def route_result(route, result, path_finder)
+        return {} if route.target?
 
-        parents(route.parent, [route.parent, *current])
+        { match: result,
+          params: path_finder.params_stack.to_h }
       end
 
-      def trailer?(route)
-        return true unless route.parent
-
-        route.parent.children.last == route
+      def route_type(route)
+        route.class.name.demodulize.underscore
       end
     end
   end

data/lib/wayfarer/cli.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    desc "version", "Print version"
+    def version
+      say Wayfarer::VERSION::STRING
+    end
+
+    class_option :require, aliases: :r, type: :string, default: nil
+
+    desc "route JOB URL", "Routing tree for URL for JOB"
+    option :format, type: :string, enum: %w[yaml json ruby], default: "yaml"
+    def route(job, url)
+      load_environment
+
+      url = parsed_url(url)
+      job = job.camelize.constantize
+      route = job.route
+
+      Wayfarer::CLI::RoutePrinter.print(route, url, format: options.fetch("format"))
+    end
+
+    desc "perform JOB URL", "Perform JOB with URL"
+    option :mock_redis, type: :boolean
+    option :batch, type: :string, default: SecureRandom.uuid
+    def perform(job, url)
+      load_environment
+      mock_redis
+
+      job = job.camelize.constantize
+      task = Wayfarer::Task.new(url, options.fetch(:batch))
+      job.new(task).perform_now
+    end
+
+    desc "enqueue JOB URL", "Enqueue JOB with URL"
+    option :batch, type: :string, default: SecureRandom.uuid
+    def enqueue(job, url)
+      load_environment
+
+      job = job.camelize.constantize
+      job.crawl(url, batch: options[:batch])
+    end
+
+    desc "execute JOB URL", "Execute JOB with async adapter starting from URL"
+    option :mock_redis, type: :boolean
+    option :batch, type: :string, default: SecureRandom.uuid
+    option :min_threads, type: :numeric, default: 1
+    option :max_threads, type: :numeric, default: 1
+    option :retain_pool, type: :boolean, default: false
+    def execute(job, url)
+      load_environment
+      mock_redis
+
+      job = job.camelize.constantize
+      job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
+                                                                     max_threads: options[:max_threads])
+      scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
+      executor = scheduler.instance_variable_get(:@async_executor)
+
+      job.crawl(url, batch: options.fetch(:batch))
+
+      sleep(0.1) while executor.scheduled_task_count > executor.completed_task_count
+
+      # Used in test suite to avoid pool recreation
+      Wayfarer::Networking::Pool.instance.free unless options.fetch(:retain_pool)
+    end
+
+  private
+
+    def mock_redis
+      Wayfarer.config[:redis][:factory] = ->(_) { MockRedis.new } if options[:mock_redis]
+    end
+
+    def parsed_url(url)
+      Addressable::URI.parse(url).normalize
+    end
+
+    def load_environment(require_path = options[:require])
+      require File.join(Dir.pwd, require_path) if require_path
+
+      load_rails
+    end
+
+    def load_rails
+      begin
+        require "rails/app_loader"
+      rescue LoadError
+        return
+      end
+
+      return unless Rails::AppLoader.find_executable
+
+      require File.expand_path("config/application", Dir.pwd)
+      require File.expand_path("config/boot", Dir.pwd)
+      require File.expand_path("config/environment", Dir.pwd)
+    end
+  end
+end

data/lib/wayfarer/gc.rb

@@ -1,14 +1,26 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  GC = Struct.new(:task) do
-    def run
-      return unless task.counter.decrement <= 0
+  # TODO: Add logging
+  module GC
+    RESETTABLES = [Wayfarer::Redis::Barrier, Wayfarer::Redis::Counter].freeze
 
-      task.metadata.job.run_callbacks(:batch)
+    class << self
+      include Wayfarer::Logging.emit(
+        after_batch: [:debug, "Running `after_batch` callback"],
+        gc: [:debug, "Garbage collecting %<resettable>s"]
+      )
+    end
+
+  module_function
+
+    def run(task)
+      task[:job].run_callbacks(:batch)
 
-      task.barrier.reset!
-      task.counter.reset!
+      RESETTABLES.each do |resettable|
+        log(:gc, task, resettable: resettable)
+        resettable.new(task).reset!
+      end
     end
   end
 end