wayfarer 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6fd1a3fe8b9b259d023e8ffb125819c8663a4c5a3ead1930b46c673198276bd
-  data.tar.gz: e98395a58d7ab9e1062319f7ffe308056d8bfeae857463f8031e463e83aed5d8
+  metadata.gz: caaf26f19cde178471b846bb33d8c347fc6ebdbb6951b9bfd40b1214516076be
+  data.tar.gz: 7347ecd76e6085091bbf21d31cccb24cffa9b0ab88ade4b8f7a388496fa345cc
 SHA512:
-  metadata.gz: d1d08a347ffc7cd54721522d7babc13ac91d50a2c06caa6bbceb13d6d633e6636e62316abd387f5c06250ed8d58802b6b766dbe2da1e8510bf638c64d05833ce
-  data.tar.gz: 185f44131bf7cebbb20eded4b83b4bf1bace6c3c41cc807e35bcda51191f6f18cba63b8dd63f963c3e24643c8eb772370cf4a6a36b00d19628853833324c4bae
+  metadata.gz: 35851a6c27663ff0013c03938accc1d144fe83f29fd29b1fba3ac6b3d436e66145291333cddeb76519ec78507409f76c74633f45adca93a7ad9fc5efdf318d2f
+  data.tar.gz: db0cbbf7a2f0524f3a1c1405e77e7136efc1256d93690ca3ea1ed1630edf7e8fbb9ecbd56a6a2aabdbba8b392965e34084ded416fcab8ffe3496087f5be7236b

data/.rubocop.yml

@@ -20,6 +20,10 @@ Style/BlockDelimiters:
     - subject
     - expect
 
+Style/ClassAndModuleChildren:
+  Exclude:
+    - "spec/**/*"
+
 Style/Documentation:
   Enabled: false
 
@@ -32,6 +36,12 @@ Metrics/BlockLength:
 
 Metrics/AbcSize:
   Max: 30
+  Exclude:
+    - "spec/**/*"
 
 Metrics/CyclomaticComplexity:
   Max: 15
+
+Lint/ConstantDefinitionInBlock:
+  Exclude:
+    - "spec/**/*"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wayfarer (0.4.3)
+    wayfarer (0.4.4)
       activejob (>= 6.0)
       addressable (~> 2.8)
       capybara (~> 3.0)
@@ -59,17 +59,16 @@ GEM
       activesupport (>= 5.0.0)
     faker (1.9.6)
       i18n (>= 0.7)
-    faraday (1.9.3)
+    faraday (1.8.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0)
-      faraday-multipart (~> 1.0)
+      faraday-httpclient (~> 1.0.1)
       faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
       faraday-patron (~> 1.0)
       faraday-rack (~> 1.0)
-      faraday-retry (~> 1.0)
+      multipart-post (>= 1.2, < 3)
       ruby2_keywords (>= 0.0.4)
     faraday-cookie_jar (0.0.7)
       faraday (>= 0.8.0)
@@ -82,22 +81,19 @@ GEM
     faraday-http-cache (2.2.0)
       faraday (>= 0.8)
     faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.3)
-      multipart-post (>= 1.2, < 3)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
-    fastimage (2.2.6)
+    fastimage (2.2.5)
     ferrum (0.11)
       addressable (~> 2.5)
       cliver (~> 0.3)
       concurrent-ruby (~> 1.1)
       websocket-driver (>= 0.6, < 0.8)
-    globalid (1.0.0)
+    globalid (0.5.2)
       activesupport (>= 5.0)
     http-cookie (1.0.4)
       domain_name (~> 0.5)
@@ -115,9 +111,9 @@ GEM
       nesty (~> 1.0)
       nokogiri (~> 1.11)
     method_source (1.0.0)
-    mime-types (3.4.1)
+    mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2022.0105)
+    mime-types-data (3.2021.0901)
     mini_mime (1.1.2)
     mini_portile2 (2.6.1)
     minitest (5.14.4)
@@ -186,7 +182,7 @@ GEM
       rack (~> 1.5)
       rack-protection (~> 1.4)
       tilt (>= 1.3, < 3)
-    thor (1.2.1)
+    thor (1.1.0)
     tilt (2.0.10)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)

data/docs/cookbook/batch_routing.md

@@ -18,5 +18,5 @@ class DummyJob < Wayfarer::Base
 end
 
 # Enqueue the task and use the database record's key as batch
-DummyJob.crawl_later("https://example.com", batch: record.id)
+DummyJob.crawl("https://example.com", batch: record.id)
 ```

data/docs/cookbook/navigation.md

@@ -0,0 +1,43 @@
+# Navigation
+
+=== "Ferrum"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        agent.goto("https://example.com")
+        agent.back
+        agent.forward
+      end
+    end
+    ```
+
+=== "Selenium"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        agent.navigate.to("https://example.com")
+        agent.navigate.back
+        agent.navigate.forward
+      end
+    end
+    ```
+
+=== "Capybara"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        agent.visit("https://example.com")
+        agent.go_back
+        agent.go_forward
+      end
+    end
+    ```

data/docs/guides/jobs.md

@@ -14,7 +14,7 @@ end
 Here is how to enqueue a task for a URL:
 
 ```ruby
-DummyJob.crawl_later("https://example.com")
+DummyJob.crawl("https://example.com")
 ```
 
 This is the same as calling the Active Job API directly and passing a task
@@ -25,10 +25,10 @@ task = Wayfarer::Task.new("https://example.com", SecureRandom.uuid)
 DummyJob.perform_later(task)
 ```
 
-A batch can be specified with `::crawl_later`, too:
+A batch can be specified with `::crawl`, too:
 
 ```ruby
-DummyJob.crawl_later("https://example.com", batch: "my-batch")
+DummyJob.crawl("https://example.com", batch: "my-batch")
 ```
 
 ## Current task
@@ -80,7 +80,7 @@ class DummyJob < Wayfarer::Base
   end
 end
 
-DummyJob.crawl_later("https://example.com/users/42/profile")
+DummyJob.crawl("https://example.com/users/42/profile")
 ```
 
 

data/docs/guides/navigation.md

@@ -67,7 +67,7 @@ class DummyJob < Wayfarer::Base
   # ...
 end
 
-DummyJob.crawl_later("https://mismatching.host/users/42")
+DummyJob.crawl("https://mismatching.host/users/42")
 ```
 
 

data/docs/guides/{browser_automation → networking}/capybara.md

@@ -60,10 +60,12 @@ end
 3. Register the driver:
 
     ```ruby
+    require "capybara/cuprite"
+
     Capybara.javascript_driver = :cuprite
 
     Capybara.register_driver(:cuprite) do |app|
-      # Wayfarer's Ferrum or Selenium options must be passed along manually
+      # Wayfarer's Ferrum or Selenium options can be passed along
       Capybara::Cuprite::Driver.new(app, Wayfarer.config.ferrum.options)
     end
     ```

data/docs/guides/networking/http.md

@@ -0,0 +1,33 @@
+# Plain HTTP
+
+Wayfarer can retrieve pages via plain HTTP requests, also alongside automated
+browsers.
+
+## Agent
+
+The HTTP agent is the default.
+
+## Ad-hoc requests
+
+When automating browsers, it can be useful to additionally retrieve the page
+over plain HTTP. Jobs can fetch URLs to [pages](/pages) with `#http`:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  route { to :index }
+
+  def index
+    http.fetch(task.url) # => #<Wayfarer::Page ...>
+  end
+end
+```
+
+By default, 3 redirects are followed, and this can be configured by passing the
+`follow` keyword:
+
+```ruby
+http.fetch(url, follow: 5)
+```
+
+If redirected too often, `Wayfarer::Networking::RedirectsExhaustedError` is
+raised.

data/lib/wayfarer/base.rb

@@ -2,49 +2,59 @@
 
 module Wayfarer
   class Base < ActiveJob::Base
-    include Wayfarer::Middleware::Worker
-    extend Forwardable
+    include Wayfarer::Middleware::Controller
+
+    use Wayfarer::Middleware::Stage
+    use Wayfarer::Middleware::Dedup
+    use Wayfarer::Middleware::Normalize
+    use Wayfarer::Middleware::Router
+    use Wayfarer::Middleware::Fetch
+    use Wayfarer::Middleware::Dispatch
+
+    ErrorHandler = lambda do |&block|
+      lambda do |job, error|
+        task = job.arguments.first
+        task.barrier.seen?(task.url)
+        task.gc.run
+        block.call(job, error)
+      end
+    end
 
-    after_enqueue do |_job|
+    after_enqueue do |job|
+      task = job.arguments.first
       task.counter.increment
     end
 
-    after_perform { |job| GC.new(job).run }
+    after_perform do |job|
+      task = job.arguments.first
+      task.gc.run
+    end
 
-    rescue_from(StandardError) { GC.new(self).run }
+    rescue_from(StandardError) do
+      task = arguments.first
+      task.gc.run
+    end
 
-    def self.retry_on(*argv)
-      super(*argv) do |job, error|
-        job.task.barrier.seen?(job.task.url)
-        GC.new(job).run
-        yield job, error if block_given?
-      end
+    def self.retry_on(*argv, &block)
+      super(*argv, &ErrorHandler.call(&block))
     end
 
-    def self.discard_on(*argv)
-      super(*argv) do |job, error|
-        job.task.barrier.seen?(job.task.url)
-        GC.new(job).run
-        yield job, error if block_given?
-      end
+    def self.discard_on(*argv, &block)
+      super(*argv, &ErrorHandler.call(&block))
     end
 
-    def self.crawl_later(url, batch: SecureRandom.uuid)
-      perform_later(Task.new(url, batch))
+    def self.crawl(url, batch: SecureRandom.uuid)
+      Task.new(url, batch).tap do |task|
+        perform_later(task)
+      end
     end
 
     def retry_job(...)
       super(...) # increments the counter by re-enqueuing the job
+      task = arguments.first
       task.counter.decrement
     end
 
-    def perform(task)
-      task.job = self
-      chain.call(task)
-    end
-
-    def task
-      arguments.first
-    end
+    alias perform call
   end
 end

data/lib/wayfarer/cli/base.rb

@@ -16,12 +16,30 @@ module Wayfarer
       end
 
       def load_environment
-        Wayfarer::CLI::Runner.loader.setup
+        load_rails || load_plain
       end
 
       def free_agent_pool
         Wayfarer::Networking::Pool.instance.free
       end
+
+      def load_rails
+        begin
+          require "rails/app_loader"
+        rescue LoadError
+          return
+        end
+
+        return unless Rails::AppLoader.find_executable
+
+        require File.expand_path("config/application", Dir.pwd)
+        require File.expand_path("config/boot", Dir.pwd)
+        require File.expand_path("config/environment", Dir.pwd)
+      end
+
+      def load_plain
+        Wayfarer::CLI::Runner.loader.setup
+      end
     end
   end
 end

data/lib/wayfarer/cli/job.rb

@@ -11,12 +11,11 @@ module Wayfarer
         load_environment
         mock_redis if options[:mock_redis]
 
-        url = Addressable::URI.parse(url)
         job = job.classify.constantize.new
         task = Wayfarer::Task.new(url, options[:batch])
         job.arguments.push(task)
         job.perform(task)
-        GC.new(job).run
+        task.gc.run
       end
 
       desc "enqueue JOB URL",
@@ -24,11 +23,10 @@ module Wayfarer
       option :batch, type: :string, default: SecureRandom.uuid
       def enqueue(job, url)
         load_environment
-        mock_redis if options[:mock_redis]
+        mock_redis if options[:mock_redis] # TODO: Remove, does not belong here
 
-        url = Addressable::URI.parse(url)
         job = job.classify.constantize
-        job.crawl_later(url, batch: options[:batch])
+        job.crawl(url, batch: options[:batch])
       end
 
       desc "execute JOB URL",
@@ -41,15 +39,13 @@ module Wayfarer
         load_environment
         mock_redis if options[:mock_redis]
 
-        url = Addressable::URI.parse(url)
         job = job.classify.constantize
-
         job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
                                                                        max_threads: options[:max_threads])
         scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
         executor = scheduler.instance_variable_get(:@async_executor)
 
-        job.crawl_later(url, batch: options[:batch])
+        job.crawl(url, batch: options[:batch])
 
         sleep(1) while executor.scheduled_task_count > executor.completed_task_count
 

data/lib/wayfarer/gc.rb

@@ -1,15 +1,14 @@
 # frozen_string_literal: true
 
 module Wayfarer
-  GC = Struct.new(:job) do
+  GC = Struct.new(:task) do
     def run
-      task = job.task
-
       return unless task.counter.decrement <= 0
 
+      task.metadata.job.run_callbacks(:batch)
+
       task.barrier.reset!
       task.counter.reset!
-      job.run_callbacks(:batch)
     end
   end
 end

data/lib/wayfarer/handler.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Handler
+    include Wayfarer::Middleware::Controller
+
+    api Wayfarer::Middleware::Fetch
+    api Wayfarer::Middleware::Stage
+
+    use Wayfarer::Middleware::Router
+    use Wayfarer::Middleware::Dispatch
+
+    singleton_class.undef_method :after_batch
+  end
+end

data/lib/wayfarer/middleware/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    module Base
+      API_MODULE = :API
+
+      def api
+        return const_get(API_MODULE) if const_defined?(API_MODULE)
+
+        Module.new
+      end
+
+      def lazy
+        Wayfarer::Middleware::Lazy.new(self)
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/chain.rb

@@ -3,6 +3,14 @@
 module Wayfarer
   module Middleware
     Chain = Struct.new(:middlewares) do
+      extend Forwardable
+
+      def self.empty
+        new([])
+      end
+
+      delegate push: :middlewares
+
       def call(task)
         traverse = lambda do |i|
           if middlewares[i]

data/lib/wayfarer/middleware/controller.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    module Controller
+      def self.included(base)
+        base.cattr_accessor :chain, default: Chain.empty
+        base.attr_accessor :task
+
+        base.extend(ClassMethods)
+        base.include(InstanceMethods)
+        base.include(Wayfarer::Callbacks)
+      end
+
+      module ClassMethods
+        def use(middleware)
+          chain.push(middleware.lazy)
+          api(middleware)
+        end
+
+        def api(middleware)
+          include middleware.api
+        end
+      end
+
+      module InstanceMethods
+        def call(task)
+          self.task = task
+
+          task.metadata.job ||= self
+          task.metadata.controller = self
+
+          self.class.chain.call(task) do
+            yield if block_given?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/dedup.rb

@@ -3,7 +3,12 @@
 module Wayfarer
   module Middleware
     class Dedup
+      extend Base
+
       def call(task)
+        # Was task routed by a previous controller already?
+        return yield if task.metadata.action
+
         return if task.barrier.seen?(task.url)
 
         begin

data/lib/wayfarer/middleware/dispatch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Dispatch
+      extend Base
+
+      def call(task)
+        controller = task.metadata.controller
+
+        controller.run_callbacks(:action) do
+          case action = task.metadata.action
+          when Symbol then controller.public_send(action)
+          else action.new.call(task)
+          end
+        end
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/fetch.rb

@@ -3,45 +3,54 @@
 module Wayfarer
   module Middleware
     class Fetch
+      extend Base
+
       module API
         def agent
-          task.metadata.agent
+          context.instance
+        end
+
+        def context
+          task.metadata.context
         end
 
         def page(live: false)
           return task.metadata.page unless live
 
-          task.metadata.page = agent.live&.page || task.metadata.page
+          task.metadata.page = context.live&.page || task.metadata.page
         end
-      end
-
-      include Wayfarer::Middleware::Stage::API
-
-      attr_reader :pool
-      attr_accessor :task
 
-      def initialize
-        @pool = Wayfarer::Networking::Pool.instance
+        def http
+          @http ||= Wayfarer::Networking::Follow.new(
+            Wayfarer::Networking::Context.new(
+              Wayfarer::Networking::HTTP.new
+            )
+          )
+        end
       end
 
       def call(task)
-        self.task = task
-
         pool.with do |context|
-          result = task.job.run_callbacks :fetch do
+          result = task.metadata.controller.run_callbacks(:fetch) do
             context.fetch(task.url)
           end
 
           case result
           when Networking::Result::Redirect
-            stage(result.redirect_url)
+            task.metadata.controller.stage(result.redirect_url)
           when Networking::Result::Success
-            task.metadata.agent = context.instance
+            task.metadata.context = context
             task.metadata.page = result.page
             yield if block_given?
           end
         end
       end
+
+    private
+
+      def pool
+        Wayfarer::Networking::Pool.instance
+      end
     end
   end
 end

data/lib/wayfarer/middleware/lazy.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    Lazy = Struct.new(:klass) do
+      def call(task, &block)
+        klass.new.call(task) { block.call }
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/normalize.rb

@@ -3,6 +3,8 @@
 module Wayfarer
   module Middleware
     class Normalize
+      extend Base
+
       def call(task)
         yield if block_given?