wayfarer 0.4.0 → 0.4.1

data/docs/reference/api/base.md

@@ -102,7 +102,7 @@ Base functionality every job is equipped with:
 
 ### `#browser -> Ferrum::Browser | Selenium::WebDriver | nil`
 :   The browser process used to retrieve the current response.
-    If the configured adapter is the default `:net_http`, `nil` is returned.
+    If the configured agent is the default `:http`, `nil` is returned.
 
     Guides:
 
@@ -112,7 +112,7 @@ Base functionality every job is equipped with:
     !!! example "Accessing a Google Chrome process"
 
         ```ruby
-        Wayfarer.config.adapter = :ferrum
+        Wayfarer.config.network.agent = :ferrum
 
         class DummyJob < Wayfarer::Base
           route.to :index
@@ -126,7 +126,7 @@ Base functionality every job is equipped with:
     !!! example "Accessing a Selenium WebDriver"
 
         ```ruby
-        Wayfarer.config.adapter = :selenium
+        Wayfarer.config.network.agent = :selenium
 
         class DummyJob < Wayfarer::Base
           route.to :index
@@ -144,7 +144,7 @@ Base functionality every job is equipped with:
     processing URL.
 
     With `page(live: true)` passed, the returned `Page` reflects the current
-    browser DOM. No-op when the `net/http` adapter is in use. Calls to
+    browser DOM. No-op when the `net/http` agent is in use. Calls to
     `page()` without the keyword return the most recent page.
 
 ---

data/docs/reference/configuration_keys.md

@@ -0,0 +1,42 @@
+---
+hide:
+  - toc
+---
+
+# Configuration Keys
+
+## `Wayfarer.config.network`
+
+| Runtime config key     | Environment variable                 | Description                                 | Default                          | Supported values                    |
+| ---------------------- | ------------------------------------ | ------------------------------------------- | -------------------------------- | ----------------------------------- |
+| `network.agent`        | `WAYFARER_NETWORK_AGENT`             | The user agent to use.                      | `:http`                          | `:http`, `:ferrum`, `:selenium`     |
+| `network.pool_size`    | `WAYFARER_NETWORK_POOL_SIZE`         | How many user agents to spawn.              | 3                                | Integers                            |
+| `network.pool_timeout` | `WAYFARER_NETWORK_POOL_TIMEOUT`      | How long jobs may use an agent in seconds.  | 10                               | Integers                            |
+| `network.http_headers` | `WAYFARER_NETWORK_HTTP_HEADERS`      | HTTP headers to append to requests.         | `{}`                             | Hashes                              |
+
+## `Wayfarer.config.ferrum`
+
+| Runtime config key     | Environment variable                 | Description                                 | Default                          | Supported values                    |
+| ---------------------- | ------------------------------------ | ------------------------------------------- | -------------------------------- | ----------------------------------- |
+| `ferrum.options`       | `WAYFARER_FERRUM_OPTIONS`            | Ferrum options.                             | `{}`                             | Hashes                              |
+
+## `Wayfarer.config.selenium`
+
+| Runtime config key        | Environment variable                 | Description                                 | Default                          | Supported values                    |
+| ----------------------    | ------------------------------------ | ------------------------------------------- | -------------------------------- | ----------------------------------- |
+| `selenium.driver`         | `WAYFARER_SELENIUM_DRIVER`           | Selenium driver to use.                     | `:chrome`                        | Symbols                             |
+| `selenium.options`        | `WAYFARER_SELENIUM_OPTIONS`          | Selenium options.                           | `{}`                             | Hashes                              |
+| `selenium.client_timeout` | `WAYFARER_SELENIUM_CLIENT_TIMEOUT`   | Selenium client timeout in seconds.         | 60                               | Integers                            |
+
+## `Wayfarer.config.redis`
+
+| Runtime config key     | Environment variable                 | Description                                 | Default                                    | Supported values                    |
+| ---------------------- | ------------------------------------ | ------------------------------------------- | ------------------------------------------ | ----------------------------------- |
+| `redis.url`            | `WAYFARER_REDIS_URL`                 | Redis URL to connect to.                    | http://localhost:6379                      | Strings                             |
+| `redis.factory`        | n/a                                  | Redis factory lambda.                       | ` ->(redis) { ::Redis.new(url: redis.url)` | Lambdas                             |
+
+## `Wayfarer.config.capybara`
+
+| Runtime config key     | Environment variable                 | Description                                 | Default                          | Supported values                    |
+| ---------------------- | ------------------------------------ | ------------------------------------------- | -------------------------------- | ----------------------------------- |
+| `capybara.driver`      | `WAYFARER_CAPYBARA_DRIVER`           | The Capybara driver to use.                 | n/a                              | Symbols                             |

data/docs/reference/environment_variables.md

@@ -10,26 +10,26 @@ with the following syntaxes:
 
 ## Variables
 
-### `WAYFARER_ADAPTER`
-:   Either `ferrum`, `selenium` or `net_http`.
+### `WAYFARER_AGENT`
+:   Either `ferrum`, `selenium` or `http`.
 
     * Type: String
-    * Key: `config.adapter`
-    * Default value: `:net_http`
+    * Key: `config.agent`
+    * Default value: `:http`
 
 ### `WAYFARER_POOL_SIZE`
-:   Number of network adapters to maintain.
+:   Number of user agents to maintain.
 
     * Type: Integer
     * Key: `config.pool_size`
     * Default value: `1`
 
 ### `WAYFARER_POOL_TIMEOUT`
-:   How long a network adapter may remain checked out until the owning job
+:   How long a user agent may remain checked out until the owning job
     fails.
 
     * Type: Integer
-    * Key: `config.adapter_pool_timeout`
+    * Key: `config.agent_pool_timeout`
     * Default value: `1`
 
 ---
@@ -43,32 +43,30 @@ with the following syntaxes:
 
 ---
 
-### `WAYFARER_SELENIUM_ARGV`
-:   Argument list passed to `Selenium::WebDriver.for`.
+### `WAYFARER_SELENIUM_DRIVER`
+:   Driver passed to `Selenium::WebDriver.for`.
 
-    * Type: Array
-    * Key: `config.selenium_argv`
-    * Default value: `[]`
+    * Type: Symbol
+    * Key: `config.selenium_driver`
+    * Default value: `:chrome`
 
-## Examples
-
-!!! example "Foobar"
-
-    For example, to run Google Chrome in foreground with Ferrum:
-
-    ```
-    Wayfarer.config.adapter = :ferrum
-    Wayfarer.ferrum_options = { headless: false, url: "http://chrome:3000" }
-    ```
+---
 
+### `WAYFARER_SELENIUM_OPTIONS`
+:   Options passed to `Selenium::WebDriver.for`.
 
-!!! example "Run Google Chrome in foreground with Ferrum"
+    * Type: Hash
+    * Key: `config.selenium_options`
+    * Default value: `{}`
+    
+---
 
-    To construct the 
+### `WAYFARER_SELENIUM_CLIENT_TIMEOUT`
+:   Selenium HTTP client timeout (seconds).
 
-    ```
-    WAYFARER_FERRUM_OPTIONS=headless:false,url:http://chrome:3000
-    ```
+    * Type: Integer
+    * Key: `config.selenium_client_timeout`
+    * Default value: `60`
 
 ---
 

data/lib/wayfarer/base.rb

@@ -5,22 +5,7 @@ module Wayfarer
     include Wayfarer::Middleware::Worker
     extend Forwardable
 
-    class << self
-      def after_batch_callbacks
-        @after_batch_callbacks ||= []
-      end
-
-      def after_batch(&block)
-        after_batch_callbacks.push(block)
-      end
-
-      def run_after_batch_callbacks
-        after_batch_callbacks.each(&:call)
-      end
-    end
-
-    after_enqueue do |job|
-      task = job.arguments.first
+    after_enqueue do |_job|
       task.counter.increment
     end
 
@@ -30,6 +15,7 @@ module Wayfarer
 
     def self.retry_on(*argv)
       super(*argv) do |job, error|
+        job.task.barrier.seen?(job.task.url)
         GC.new(job).run
         yield job, error if block_given?
       end
@@ -37,6 +23,7 @@ module Wayfarer
 
     def self.discard_on(*argv)
       super(*argv) do |job, error|
+        job.task.barrier.seen?(job.task.url)
         GC.new(job).run
         yield job, error if block_given?
       end
@@ -48,7 +35,6 @@ module Wayfarer
 
     def retry_job(...)
       super(...) # increments the counter by re-enqueuing the job
-      task = arguments.first
       task.counter.decrement
     end
 
@@ -56,5 +42,9 @@ module Wayfarer
       task.job = self
       chain.call(task)
     end
+
+    def task
+      arguments.first
+    end
   end
 end

data/lib/wayfarer/callbacks.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Callbacks
+    TERMINATOR = ->(_target, result) { result.call == false }
+    OPTIONS = { terminator: TERMINATOR, skip_after_callbacks_if_terminated: true }.freeze
+
+    ConditionalCallback = Struct.new(:job, :filters) do
+      def run(method, &block)
+        return if only && !applies?(only)
+        return if except && applies?(except)
+
+        return job.send(method) if method
+
+        job.instance_eval(&block)
+      end
+
+    private
+
+      def applies?(condition)
+        case condition
+        when Symbol then condition == action
+        when Enumerable then condition&.include?(action)
+        end
+      end
+
+      def only
+        filters[:only]
+      end
+
+      def except
+        filters[:except]
+      end
+
+      def action
+        task.metadata.action
+      end
+
+      def task
+        job.task
+      end
+    end
+
+    def self.included(base)
+      base.include(ActiveSupport::Callbacks)
+      base.extend(ClassMethods)
+
+      base.class_eval do
+        define_callbacks(:fetch, OPTIONS)
+        define_callbacks(:action, OPTIONS)
+        define_callbacks(:batch, OPTIONS)
+
+        define(:fetch, :before)
+        define(:action, :before)
+        define(:batch, :after)
+      end
+    end
+
+    module ClassMethods
+    private
+
+      def define(name, stage)
+        define_singleton_method([stage, name].join("_")) do |method = nil, **filters, &block|
+          set_callback(name, stage, **filters) do |job|
+            ConditionalCallback.new(job, filters).run(method, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/base.rb

@@ -12,12 +12,16 @@ module Wayfarer
     private
 
       def mock_redis
-        Wayfarer.config.redis_factory = -> { MockRedis.new }
+        Wayfarer.config.redis.factory = ->(_) { MockRedis.new }
       end
 
       def load_environment
         Wayfarer::CLI::Runner.loader.setup
       end
+
+      def free_agent_pool
+        Wayfarer::Networking::Pool.instance.free
+      end
     end
   end
 end

data/lib/wayfarer/cli/job.rb

@@ -11,12 +11,14 @@ module Wayfarer
         load_environment
         mock_redis if options[:mock_redis]
 
-        url = URI(url)
+        url = Addressable::URI.parse(url)
         job = job.classify.constantize.new
         task = Wayfarer::Task.new(url, "tmp")
         job.arguments.push(task)
         job.perform(task)
         GC.new(job).run
+
+        free_agent_pool
       end
 
       desc "enqueue JOB URL",
@@ -26,7 +28,7 @@ module Wayfarer
         load_environment
         mock_redis if options[:mock_redis]
 
-        url = URI(url)
+        url = Addressable::URI.parse(url)
         job = job.classify.constantize
         job.crawl_later(url, batch: options[:batch])
       end
@@ -41,7 +43,7 @@ module Wayfarer
         load_environment
         mock_redis if options[:mock_redis]
 
-        url = URI(url)
+        url = Addressable::URI.parse(url)
         job = job.classify.constantize
 
         job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
@@ -52,6 +54,8 @@ module Wayfarer
         job.crawl_later(url, batch: options[:batch])
 
         sleep(1) while executor.scheduled_task_count > executor.completed_task_count
+
+        free_agent_pool
       end
     end
   end

data/lib/wayfarer/cli/route.rb

@@ -9,7 +9,7 @@ module Wayfarer
            "Invoke JOB's router with URL"
       def result(job, url)
         load_environment
-        url = URI(url)
+        url = Addressable::URI.parse(url)
         job = job.classify.constantize
         puts Wayfarer::Routing::PathFinder.result(job.route, url)
       end
@@ -18,7 +18,7 @@ module Wayfarer
            "Visualize JOB's routing tree for URL"
       def tree(job, url)
         load_environment
-        url = URI(url)
+        url = Addressable::URI.parse(url)
         job = job.classify.constantize
         Wayfarer::CLI::RoutePrinter.print(job.route, url)
       end

data/lib/wayfarer/cli/route_printer.rb

@@ -77,19 +77,19 @@ module Wayfarer
       def matcher_label(route)
         return "Target" if route.is_a?(Wayfarer::Routing::TargetRoute)
 
-        route.matcher.class.name.demodulize.delete_suffix("Matcher")
+        route.matcher.class.name.demodulize
       end
 
       def options(route)
         return "" if route.is_a?(Wayfarer::Routing::RootRoute)
 
         case (matcher = route.matcher)
-        when Wayfarer::Routing::HostMatcher then matcher.host
-        when Wayfarer::Routing::PathMatcher then matcher.path
-        when Wayfarer::Routing::QueryMatcher then matcher.fields
-        when Wayfarer::Routing::CustomMatcher then "##{route.action}"
-        when Wayfarer::Routing::SchemeMatcher then matcher.scheme
-        when Wayfarer::Routing::SuffixMatcher then matcher.suffix
+        when Wayfarer::Routing::Matchers::Host then matcher.host
+        when Wayfarer::Routing::Matchers::Path then matcher.path
+        when Wayfarer::Routing::Matchers::Query then matcher.fields
+        when Wayfarer::Routing::Matchers::Custom then "##{route.action}"
+        when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
+        when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
         end
       end
 

data/lib/wayfarer/config/capybara.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Capybara = Struct.new(driver: {
+                            env_key: "WAYFARER_CAPYBARA_DRIVER",
+                            type: Symbol
+                          })
+  end
+end

data/lib/wayfarer/config/ferrum.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Ferrum = Struct.new(options: {
+                          env_key: "WAYFARER_FERRUM_OPTIONS",
+                          type: Hash,
+                          default: {}
+                        })
+  end
+end

data/lib/wayfarer/config/networking.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Networking = Struct.new(agent: {
+                              env_key: "WAYFARER_NETWORK_AGENT",
+                              type: Symbol,
+                              default: :http
+                            },
+                            pool_size: {
+                              env_key: "WAYFARER_NETWORK_POOL_SIZE",
+                              type: Integer,
+                              default: 3
+                            },
+                            pool_timeout: {
+                              env_key: "WAYFARER_NETWORK_POOL_TIMEOUT",
+                              type: Integer,
+                              default: 10
+                            },
+                            http_headers: {
+                              env_key: "WAYFARER_NETWORK_HTTP_HEADERS",
+                              type: Hash,
+                              default: {}
+                            })
+  end
+end

data/lib/wayfarer/config/redis.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Redis = Struct.new(url: {
+                         env_key: "WAYFARER_REDIS_URL",
+                         type: String,
+                         default: "redis://localhost:6379"
+                       },
+                       factory: {
+                         default: ->(redis) { ::Redis.new(url: redis.url) }
+                       })
+  end
+end

data/lib/wayfarer/config/root.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Root = Struct.new(ferrum: { default: Wayfarer::Config::Ferrum.new },
+                      network: { default: Wayfarer::Config::Networking.new },
+                      redis: { default: Wayfarer::Config::Redis.new },
+                      selenium: { default: Wayfarer::Config::Selenium.new },
+                      capybara: { default: Wayfarer::Config::Capybara.new })
+  end
+end

data/lib/wayfarer/config/selenium.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Selenium = Struct.new(driver: {
+                            env_key: "WAYFARER_SELENIUM_DRIVER",
+                            type: Symbol,
+                            default: :chrome
+                          },
+                          options: {
+                            env_key: "WAYFARER_SELENIUM_OPTIONS",
+                            type: Hash,
+                            default: {}
+                          },
+                          client_timeout: {
+                            env_key: "WAYFARER_SELENIUM_CLIENT_TIMEOUT",
+                            type: Integer,
+                            default: 60 # seconds
+                          })
+  end
+end

data/lib/wayfarer/config/strconv.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    module Strconv
+    module_function
+
+      def parse(str, type = nil)
+        return primitive(str) unless type
+
+        case type.name
+        when "Hash" then hash(str)
+        when "Array" then array(str)
+        when "Symbol" then str.to_sym
+        when "Integer" then Integer(str)
+        else str
+        end
+      end
+
+      def hash(str)
+        array(str).reduce({}) do |acc, pair|
+          k, v = pair.split(":", 2)
+          next acc unless k && v
+
+          acc.merge({ parse(k, Symbol) => primitive(v) })
+        end
+      end
+
+      def array(str)
+        str.split(",").map(&:strip)
+      end
+
+      def primitive(str)
+        return true if str == "true"
+        return false if str == "false"
+
+        begin
+          parse(str, Integer)
+        rescue StandardError
+          str
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/config/struct.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    class Struct
+      module ClassMethods
+        attr_accessor :members
+      end
+
+      module InstanceMethods
+        extend Forwardable
+
+        delegate members: "self.class"
+
+        attr_reader :env
+
+        def initialize(env = ENV)
+          @env = env
+
+          define_writers
+          define_readers
+        end
+
+      private
+
+        def define_writers
+          members.each { |key, _| define_writer(key) }
+        end
+
+        def define_writer(key)
+          define_singleton_method(:"#{key}=") do |val|
+            set(key, val)
+          end
+        end
+
+        def define_readers
+          members.each { |key, options| define_reader(key, **options) }
+        end
+
+        def define_reader(key, env_key: nil, type: nil, default: nil)
+          define_singleton_method(key.to_sym) do
+            get(key) || set(key, get(key) || env_val(env_key, type) || default)
+          end
+        end
+
+        def env_val(env_key, type)
+          return nil unless env_key
+          return nil unless env.key?(env_key)
+
+          Strconv.parse(env[env_key], type)
+        end
+
+        def get(key)
+          instance_variable_get(:"@#{key}")
+        end
+
+        def set(key, val)
+          instance_variable_set(:"@#{key}", val)
+        end
+      end
+
+      def self.new(members)
+        Class.new do
+          include InstanceMethods
+          extend ClassMethods
+
+          self.members = members
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/gc.rb

@@ -3,17 +3,13 @@
 module Wayfarer
   GC = Struct.new(:job) do
     def run
+      task = job.task
+
       return unless task.counter.decrement <= 0
 
       task.barrier.reset!
       task.counter.reset!
-      job.class.run_after_batch_callbacks
-    end
-
-  private
-
-    def task
-      job.arguments.first
+      job.run_callbacks(:batch)
     end
   end
 end

data/lib/wayfarer/middleware/fetch.rb

@@ -15,10 +15,14 @@ module Wayfarer
       def call(task)
         self.task = task
 
-        pool.with do |adapter|
-          task.metadata.adapter = adapter
+        pool.with do |agent|
+          task.metadata.agent = agent
 
-          case result = adapter.fetch(task.url)
+          result = task.job.run_callbacks :fetch do
+            agent.fetch(task.url)
+          end
+
+          case result
           when Networking::Result::Redirect
             stage(result.redirect_url)
           when Networking::Result::Success

data/lib/wayfarer/middleware/router.rb

@@ -6,12 +6,12 @@ module Wayfarer
       def call(task)
         route = task.job.class.route
 
-        case result = route.invoke(URI(task.url))
+        case result = route.invoke(Addressable::URI.parse(task.url))
         when Routing::Result::Mismatch
           return
         when Routing::Result::Match
           task.metadata.action = result.action
-          task.metadata.params = result.params
+          task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new(result.params)
         end
 
         yield if block_given?