wayfarer 0.4.1 → 0.4.5

data/lib/wayfarer/cli/base.rb

@@ -16,12 +16,30 @@ module Wayfarer
       end
 
       def load_environment
-        Wayfarer::CLI::Runner.loader.setup
+        load_rails || load_plain
       end
 
       def free_agent_pool
         Wayfarer::Networking::Pool.instance.free
       end
+
+      def load_rails
+        begin
+          require "rails/app_loader"
+        rescue LoadError
+          return
+        end
+
+        return unless Rails::AppLoader.find_executable
+
+        require File.expand_path("config/application", Dir.pwd)
+        require File.expand_path("config/boot", Dir.pwd)
+        require File.expand_path("config/environment", Dir.pwd)
+      end
+
+      def load_plain
+        Wayfarer::CLI::Runner.loader.setup
+      end
     end
   end
 end

data/lib/wayfarer/cli/job.rb

@@ -11,14 +11,11 @@ module Wayfarer
         load_environment
         mock_redis if options[:mock_redis]
 
-        url = Addressable::URI.parse(url)
         job = job.classify.constantize.new
-        task = Wayfarer::Task.new(url, "tmp")
+        task = Wayfarer::Task.new(url, options[:batch])
         job.arguments.push(task)
         job.perform(task)
-        GC.new(job).run
-
-        free_agent_pool
+        task.gc.run
       end
 
       desc "enqueue JOB URL",
@@ -26,11 +23,10 @@ module Wayfarer
       option :batch, type: :string, default: SecureRandom.uuid
       def enqueue(job, url)
         load_environment
-        mock_redis if options[:mock_redis]
+        mock_redis if options[:mock_redis] # TODO: Remove, does not belong here
 
-        url = Addressable::URI.parse(url)
         job = job.classify.constantize
-        job.crawl_later(url, batch: options[:batch])
+        job.crawl(url, batch: options[:batch])
       end
 
       desc "execute JOB URL",
@@ -43,15 +39,13 @@ module Wayfarer
         load_environment
         mock_redis if options[:mock_redis]
 
-        url = Addressable::URI.parse(url)
         job = job.classify.constantize
-
         job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
                                                                        max_threads: options[:max_threads])
         scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
         executor = scheduler.instance_variable_get(:@async_executor)
 
-        job.crawl_later(url, batch: options[:batch])
+        job.crawl(url, batch: options[:batch])
 
         sleep(1) while executor.scheduled_task_count > executor.completed_task_count
 

data/lib/wayfarer/cli/route.rb

@@ -11,7 +11,8 @@ module Wayfarer
         load_environment
         url = Addressable::URI.parse(url)
         job = job.classify.constantize
-        puts Wayfarer::Routing::PathFinder.result(job.route, url)
+        job.router.invoke(url, job.new.steer)
+        say Wayfarer::Routing::PathFinder.result(job.router.root, url)
       end
 
       desc "tree JOB URL",
@@ -20,7 +21,8 @@ module Wayfarer
         load_environment
         url = Addressable::URI.parse(url)
         job = job.classify.constantize
-        Wayfarer::CLI::RoutePrinter.print(job.route, url)
+        job.router.invoke(url, job.new.steer)
+        Wayfarer::CLI::RoutePrinter.print(job.router.root, url)
       end
     end
   end

data/lib/wayfarer/cli/templates/job.rb.tt

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 class <%= @name.camelize %> < Wayfarer::Base
-  route.to :index
+  route { to :index }
+
+  retry_on ConnectionPool::TimeoutError, attempts: 3
 
   def index
   end

data/lib/wayfarer/config/networking.rb

@@ -10,7 +10,7 @@ module Wayfarer
                             pool_size: {
                               env_key: "WAYFARER_NETWORK_POOL_SIZE",
                               type: Integer,
-                              default: 3
+                              default: 1
                             },
                             pool_timeout: {
                               env_key: "WAYFARER_NETWORK_POOL_TIMEOUT",

data/lib/wayfarer/config/struct.rb

@@ -39,7 +39,7 @@ module Wayfarer
 
         def define_reader(key, env_key: nil, type: nil, default: nil)
           define_singleton_method(key.to_sym) do
-            get(key) || set(key, get(key) || env_val(env_key, type) || default)
+            get(key) || set(key, env_val(env_key, type) || default)
           end
         end
 

data/lib/wayfarer/gc.rb

@@ -1,15 +1,14 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  GC = Struct.new(:job) do
+  GC = Struct.new(:task) do
     def run
-      task = job.task
-
       return unless task.counter.decrement <= 0
 
+      task.metadata.job.run_callbacks(:batch)
+
       task.barrier.reset!
       task.counter.reset!
-      job.run_callbacks(:batch)
     end
   end
 end

data/lib/wayfarer/handler.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Handler
+    include Wayfarer::Middleware::Controller
+
+    api Wayfarer::Middleware::Fetch
+    api Wayfarer::Middleware::Stage
+
+    use Wayfarer::Middleware::Router
+    use Wayfarer::Middleware::Dispatch
+
+    singleton_class.undef_method :after_batch
+  end
+end

data/lib/wayfarer/middleware/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    module Base
+      API_MODULE = :API
+
+      def api
+        return const_get(API_MODULE) if const_defined?(API_MODULE)
+
+        Module.new
+      end
+
+      def lazy
+        Wayfarer::Middleware::Lazy.new(self)
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/chain.rb

@@ -3,6 +3,14 @@
 module Wayfarer
   module Middleware
     Chain = Struct.new(:middlewares) do
+      extend Forwardable
+
+      def self.empty
+        new([])
+      end
+
+      delegate push: :middlewares
+
       def call(task)
         traverse = lambda do |i|
           if middlewares[i]

data/lib/wayfarer/middleware/controller.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    module Controller
+      def self.included(base)
+        base.cattr_accessor :chain, default: Chain.empty
+        base.attr_accessor :task
+
+        base.extend(ClassMethods)
+        base.include(InstanceMethods)
+        base.include(Wayfarer::Callbacks)
+      end
+
+      module ClassMethods
+        def use(middleware)
+          chain.push(middleware.lazy)
+          api(middleware)
+        end
+
+        def api(middleware)
+          include middleware.api
+        end
+      end
+
+      module InstanceMethods
+        def call(task)
+          self.task = task
+
+          task.metadata.job ||= self
+          task.metadata.controller = self
+
+          self.class.chain.call(task) do
+            yield if block_given?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/dedup.rb

@@ -3,7 +3,12 @@
 module Wayfarer
   module Middleware
     class Dedup
+      extend Base
+
       def call(task)
+        # Was task routed by a previous controller already?
+        return yield if task.metadata.action
+
         return if task.barrier.seen?(task.url)
 
         begin

data/lib/wayfarer/middleware/dispatch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Dispatch
+      extend Base
+
+      def call(task)
+        controller = task.metadata.controller
+
+        controller.run_callbacks(:action) do
+          case action = task.metadata.action
+          when Symbol then controller.public_send(action)
+          else action.new.call(task)
+          end
+        end
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/fetch.rb

@@ -3,34 +3,54 @@
 module Wayfarer
   module Middleware
     class Fetch
-      include Wayfarer::Middleware::Stage::API
+      extend Base
 
-      attr_reader :pool
-      attr_accessor :task
+      module API
+        def agent
+          context.instance
+        end
 
-      def initialize
-        @pool = Wayfarer::Networking::Pool.instance
-      end
+        def context
+          task.metadata.context
+        end
 
-      def call(task)
-        self.task = task
+        def page(live: false)
+          return task.metadata.page unless live
+
+          task.metadata.page = context.live&.page || task.metadata.page
+        end
 
-        pool.with do |agent|
-          task.metadata.agent = agent
+        def http
+          @http ||= Wayfarer::Networking::Follow.new(
+            Wayfarer::Networking::Context.new(
+              Wayfarer::Networking::HTTP.new
+            )
+          )
+        end
+      end
 
-          result = task.job.run_callbacks :fetch do
-            agent.fetch(task.url)
+      def call(task)
+        pool.with do |context|
+          result = task.metadata.controller.run_callbacks(:fetch) do
+            context.fetch(task.url)
           end
 
           case result
           when Networking::Result::Redirect
-            stage(result.redirect_url)
+            task.metadata.controller.stage(result.redirect_url)
           when Networking::Result::Success
+            task.metadata.context = context
             task.metadata.page = result.page
             yield if block_given?
           end
         end
       end
+
+    private
+
+      def pool
+        Wayfarer::Networking::Pool.instance
+      end
     end
   end
 end

data/lib/wayfarer/middleware/lazy.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    Lazy = Struct.new(:klass) do
+      def call(task, &block)
+        klass.new.call(task) { block.call }
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/normalize.rb

@@ -3,6 +3,8 @@
 module Wayfarer
   module Middleware
     class Normalize
+      extend Base
+
       def call(task)
         yield if block_given?
 

data/lib/wayfarer/middleware/router.rb

@@ -3,15 +3,53 @@
 module Wayfarer
   module Middleware
     class Router
+      extend Base
+
+      module API
+        def self.included(base)
+          base.extend(ClassMethods)
+          base.include(InstanceMethods)
+        end
+
+        module ClassMethods
+          def router
+            # TODO: Use cattr_accessor
+            @router ||= Wayfarer::Routing::Router.new
+          end
+
+          def route(&block)
+            router.draw(&block) if block_given?
+          end
+
+          def steer(&block)
+            define_method(:steer) { block.call(task) }
+          end
+        end
+
+        module InstanceMethods
+          def steer
+            []
+          end
+
+          def params
+            task.metadata.params
+          end
+        end
+      end
+
       def call(task)
-        route = task.job.class.route
+        controller = task.metadata.controller
+        # TODO: The router has to be cloned because it's not thread-safe
+        router = controller.class.router.clone
+        url = Addressable::URI.parse(task.url)
 
-        case result = route.invoke(Addressable::URI.parse(task.url))
+        case result = router.invoke(url, controller.steer)
         when Routing::Result::Mismatch
           return
         when Routing::Result::Match
           task.metadata.action = result.action
-          task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new(result.params)
+          task.metadata.params ||= ActiveSupport::HashWithIndifferentAccess.new
+          task.metadata.params.merge!(result.params)
         end
 
         yield if block_given?

data/lib/wayfarer/middleware/stage.rb

@@ -3,9 +3,11 @@
 module Wayfarer
   module Middleware
     class Stage
+      extend Base
+
       module API
         def stage(urls)
-          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url) }
+          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url.to_s) }
         end
       end
 
@@ -15,8 +17,10 @@ module Wayfarer
         yield if block_given?
 
         task.metadata.staged_urls.each do |url|
-          task.job.class.crawl_later(url, batch: task.batch)
+          task.metadata.job.class.crawl(url, batch: task.batch)
         end
+
+        task.metadata.staged_urls.clear
       end
     end
   end

data/lib/wayfarer/networking/context.rb

@@ -29,6 +29,7 @@ module Wayfarer
         renew
       ensure
         # If renewing raises, re-raise the originally caught exception
+        # TODO: Not nice this effectively swallows exceptions
         raise e if e
       end
     end

data/lib/wayfarer/networking/ferrum.rb

@@ -22,6 +22,7 @@ module Wayfarer
 
       def navigate(instance, url)
         instance.goto(url)
+        # instance.network.wait_for_idle
       end
 
       def live(instance)

data/lib/wayfarer/networking/follow.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    RedirectsExhaustedError = Class.new(StandardError)
+
+    Follow = Struct.new(:context) do
+      extend Forwardable
+
+      delegate %i[live renew instance] => :context
+
+      def fetch(url, follow: 3)
+        raise RedirectsExhaustedError if follow.negative?
+
+        case result = context.fetch(url)
+        when Result::Success then result.page
+        when Result::Redirect then fetch(result.redirect_url, follow: follow - 1)
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/pool.rb

@@ -4,22 +4,23 @@ module Wayfarer
   module Networking
     class Pool
       include Singleton
-      extend Forwardable
 
       cattr_accessor :registry, default: { http: HTTP,
                                            ferrum: Ferrum,
                                            selenium: Selenium,
                                            capybara: Capybara }
 
-      attr_reader :pool
-
-      def initialize
-        @pool = ConnectionPool.new(size: Wayfarer.config.network.pool_size,
-                                   timeout: Wayfarer.config.network.pool_timeout,
-                                   &method(:context))
+      def pool
+        @pool ||= ConnectionPool.new(size: Wayfarer.config.network.pool_size,
+                                     timeout: Wayfarer.config.network.pool_timeout,
+                                     &method(:context))
       end
 
-      delegate with: :pool
+      def with(&block)
+        pool.with(&block)
+      rescue ConnectionPool::TimeoutError => e
+        raise Wayfarer::UserAgentTimeoutError, e
+      end
 
       def free
         pool.shutdown(&:renew)

data/lib/wayfarer/networking/result.rb

@@ -2,16 +2,8 @@
 
 module Wayfarer
   module Networking
-    # Wrapper structs for page retrieval results.
     module Result
-      # Signals that a page has been retrieved (2xx or no browser error).
-      # @!attribute [rw] page
-      #   @return [Page] the retrieved page.
       Success = Struct.new(:page)
-
-      # Signals that a URL resulted in response with 3xx.
-      # @!attribute [rw] redirect_url
-      #   @return [URI] Where to go next.
       Redirect = Struct.new(:redirect_url)
     end
   end

data/lib/wayfarer/page.rb

@@ -20,7 +20,7 @@ module Wayfarer
       # If no Content-Type field is present, assume HTML/XML
       return @doc = Wayfarer::Parsing::XML.parse_html(body) unless headers["content-type"]
 
-      content_type = headers["content-type"].first
+      content_type = headers["content-type"]
       sub_type = MIME::Types[content_type].first.sub_type
 
       @doc = case sub_type

data/lib/wayfarer/routing/matchers/custom.rb

@@ -4,6 +4,8 @@ module Wayfarer
   module Routing
     module Matchers
       class Custom
+        include Stringify
+
         attr_reader :delegate
 
         def initialize(delegate = proc)

data/lib/wayfarer/routing/matchers/host.rb

@@ -6,7 +6,9 @@ module Wayfarer
       Host = Struct.new(:host) do
         # rubocop:disable Style/CaseEquality
         def match(url)
-          host === url.host
+          # url.host excludes the port
+          # TODO: Test case, docs
+          host === url.authority
         end
         # rubocop:enable Style/CaseEquality
 

data/lib/wayfarer/routing/matchers/path.rb

@@ -13,7 +13,7 @@ module Wayfarer
           @path = path
           @route = route
           @peeking = false
-          @matcher = Mustermann.new(path, type: "sinatra")
+          @matcher = Mustermann.new(path, type: "sinatra") # TODO: Add type constant
         end
 
         def match(url)
@@ -36,6 +36,7 @@ module Wayfarer
 
         def visit(route)
           return true if route == self.route
+          return true if route.is_a?(TargetRoute)
 
           return unless route.matcher.is_a?(self.class)
 

data/lib/wayfarer/routing/result.rb

@@ -3,12 +3,7 @@
 module Wayfarer
   module Routing
     module Result
-      # Signals that a URL matched a route.
-      # @!attribute [rw] action
-      #   @return [Page] the retrieved page.
       Match = Struct.new(:action, :params)
-
-      # Signals that a URL matched no route.
       Mismatch = Class.new
     end
   end

data/lib/wayfarer/routing/route.rb

@@ -4,6 +4,7 @@ module Wayfarer
   module Routing
     class Route
       include DSL
+      include Stringify
 
       attr_reader :children
 
@@ -12,6 +13,11 @@ module Wayfarer
                     :action,
                     :path_offset
 
+      stringify :matcher,
+                :action,
+                :path_offset,
+                :children
+
       def initialize(matcher = Matchers::Custom.new { children.any? }, path_offset = "/")
         @matcher = matcher
         @children = []

data/lib/wayfarer/routing/router.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class Router
+      ArgumentCountError = Class.new(StandardError) # TODO: Unused, remove
+
+      extend Forwardable
+
+      attr_reader :root,
+                  :blocks
+
+      def initialize
+        @blocks = []
+      end
+
+      def draw(&block)
+        @blocks.push(block)
+      end
+
+      def invoke(url, arguments)
+        @root = Wayfarer::Routing::RootRoute.new
+        @blocks.each { |block| Docile.dsl_eval(@root, *arguments, &block) }
+        root.invoke(url)
+      end
+    end
+  end
+end

data/lib/wayfarer/serializer.rb

@@ -7,11 +7,11 @@ module Wayfarer
     end
 
     def serialize(task)
-      super(url: task.url, batch: task.batch)
+      super("url" => task.url, "batch" => task.batch)
     end
 
     def deserialize(hash)
-      Wayfarer::Task.new(*hash.values_at(:url, :batch))
+      Wayfarer::Task.new(*hash.values_at("url", "batch"))
     end
   end
 end