wayfarer 0.4.0 → 0.4.1

data/wayfarer.gemspec

@@ -2,12 +2,12 @@
 
 Gem::Specification.new do |s|
   s.name = "wayfarer"
-  s.version = "0.4.0"
+  s.version = "0.4.1"
   s.license = "MIT"
 
   s.homepage = "http://github.com/bauerd/wayfarer"
-  s.description = "Versatile web crawling with Ruby"
-  s.summary = s.description
+  s.summary = "Versatile web crawling with Ruby"
+  s.description = "Web crawling framework based on ActiveJob"
 
   s.date = "2021-09-26"
   s.authors = ["Dominic Bauer"]
@@ -23,8 +23,8 @@ Gem::Specification.new do |s|
   s.required_ruby_version = "~> 2.7"
 
   s.add_runtime_dependency "activejob", "~> 6.0"
-  s.add_runtime_dependency "capybara", "~> 2.5"
-  s.add_runtime_dependency "cuprite", "~> 0.11"
+  s.add_runtime_dependency "addressable", "~> 2.8"
+  s.add_runtime_dependency "capybara", "~> 3.0"
   s.add_runtime_dependency "connection_pool", "~> 2.2"
   s.add_runtime_dependency "docile", "~> 1.1"
   s.add_runtime_dependency "ferrum", "~> 0.9"
@@ -32,13 +32,14 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "mime-types", "~> 3.0"
   s.add_runtime_dependency "mustermann", "~> 1.1"
   s.add_runtime_dependency "mock_redis", "~> 0.29"
-  s.add_runtime_dependency "net-http-persistent", "~> 2.9"
+  s.add_runtime_dependency "net-http-persistent", "~> 3.0"
   s.add_runtime_dependency "nokogiri", "~> 1.11"
   s.add_runtime_dependency "normalize_url", "~> 0.0.6"
-  s.add_runtime_dependency "redis", "~> 4.0"
+  s.add_runtime_dependency "redis", "~> 4.4", "< 4.5" # redis 4.5.x broke SMISMEMBER
   s.add_runtime_dependency "selenium-webdriver", "~> 3.4"
   s.add_runtime_dependency "thor", "~> 1.0"
 
+  s.add_development_dependency "cuprite", "~> 0.13"
   s.add_development_dependency "factory_bot", "~> 6.0"
   s.add_development_dependency "faker", "~> 1.7"
   s.add_development_dependency "pry", "~> 0.10"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayfarer
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Dominic Bauer
@@ -25,33 +25,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
-  name: capybara
+  name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
-  name: cuprite
+  name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +156,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.5'
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +240,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: cuprite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: factory_bot
   requirement: !ruby/object:Gem::Requirement
@@ -346,7 +366,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-description: Versatile web crawling with Ruby
+description: Web crawling framework based on ActiveJob
 email: mail@bauerd.com
 executables:
 - wayfarer
@@ -372,6 +392,7 @@ files:
 - docs/cookbook/screenshots.md
 - docs/cookbook/user_agent.md
 - docs/guides/browser_automation/capybara.md
+- docs/guides/browser_automation/custom_adapters.md
 - docs/guides/browser_automation/ferrum.md
 - docs/guides/browser_automation/selenium.md
 - docs/guides/callbacks.md
@@ -386,9 +407,11 @@ files:
 - docs/reference/api/base.md
 - docs/reference/api/route.md
 - docs/reference/cli.md
+- docs/reference/configuration_keys.md
 - docs/reference/environment_variables.md
 - lib/wayfarer.rb
 - lib/wayfarer/base.rb
+- lib/wayfarer/callbacks.rb
 - lib/wayfarer/cli/base.rb
 - lib/wayfarer/cli/generate.rb
 - lib/wayfarer/cli/job.rb
@@ -397,7 +420,14 @@ files:
 - lib/wayfarer/cli/runner.rb
 - lib/wayfarer/cli/templates/Gemfile.tt
 - lib/wayfarer/cli/templates/job.rb.tt
-- lib/wayfarer/config.rb
+- lib/wayfarer/config/capybara.rb
+- lib/wayfarer/config/ferrum.rb
+- lib/wayfarer/config/networking.rb
+- lib/wayfarer/config/redis.rb
+- lib/wayfarer/config/root.rb
+- lib/wayfarer/config/selenium.rb
+- lib/wayfarer/config/strconv.rb
+- lib/wayfarer/config/struct.rb
 - lib/wayfarer/gc.rb
 - lib/wayfarer/middleware/chain.rb
 - lib/wayfarer/middleware/dedup.rb
@@ -406,12 +436,14 @@ files:
 - lib/wayfarer/middleware/router.rb
 - lib/wayfarer/middleware/stage.rb
 - lib/wayfarer/middleware/worker.rb
+- lib/wayfarer/networking/capybara.rb
+- lib/wayfarer/networking/context.rb
 - lib/wayfarer/networking/ferrum.rb
-- lib/wayfarer/networking/healer.rb
-- lib/wayfarer/networking/net_http.rb
+- lib/wayfarer/networking/http.rb
 - lib/wayfarer/networking/pool.rb
 - lib/wayfarer/networking/result.rb
 - lib/wayfarer/networking/selenium.rb
+- lib/wayfarer/networking/strategy.rb
 - lib/wayfarer/page.rb
 - lib/wayfarer/parsing/json.rb
 - lib/wayfarer/parsing/xml.rb
@@ -420,29 +452,37 @@ files:
 - lib/wayfarer/redis/counter.rb
 - lib/wayfarer/redis/pool.rb
 - lib/wayfarer/redis/version.rb
-- lib/wayfarer/routing/custom_matcher.rb
 - lib/wayfarer/routing/dsl.rb
-- lib/wayfarer/routing/host_matcher.rb
+- lib/wayfarer/routing/matchers/custom.rb
+- lib/wayfarer/routing/matchers/host.rb
+- lib/wayfarer/routing/matchers/path.rb
+- lib/wayfarer/routing/matchers/query.rb
+- lib/wayfarer/routing/matchers/scheme.rb
+- lib/wayfarer/routing/matchers/suffix.rb
+- lib/wayfarer/routing/matchers/url.rb
 - lib/wayfarer/routing/path_finder.rb
-- lib/wayfarer/routing/path_matcher.rb
-- lib/wayfarer/routing/query_matcher.rb
 - lib/wayfarer/routing/result.rb
 - lib/wayfarer/routing/root_route.rb
 - lib/wayfarer/routing/route.rb
-- lib/wayfarer/routing/scheme_matcher.rb
-- lib/wayfarer/routing/suffix_matcher.rb
 - lib/wayfarer/routing/target_route.rb
-- lib/wayfarer/routing/url_matcher.rb
 - lib/wayfarer/serializer.rb
 - lib/wayfarer/stringify.rb
 - lib/wayfarer/task.rb
 - mkdocs.yml
 - requirements.txt
 - spec/base_spec.rb
+- spec/callbacks_spec.rb
 - spec/cli/generate_spec.rb
 - spec/cli/job_spec.rb
 - spec/cli/version_spec.rb
-- spec/config_spec.rb
+- spec/config/capybara_spec.rb
+- spec/config/ferrum_spec.rb
+- spec/config/networking_spec.rb
+- spec/config/redis_spec.rb
+- spec/config/root_spec.rb
+- spec/config/selenium_spec.rb
+- spec/config/strconv_spec.rb
+- spec/config/struct_spec.rb
 - spec/factories/queue/chain.rb
 - spec/factories/queue/middleware.rb
 - spec/factories/queue/page.rb
@@ -456,12 +496,13 @@ files:
 - spec/middleware/router_spec.rb
 - spec/middleware/stage_spec.rb
 - spec/middleware/worker_spec.rb
-- spec/networking/adapter.rb
+- spec/networking/capybara_spec.rb
+- spec/networking/context_spec.rb
 - spec/networking/ferrum_spec.rb
-- spec/networking/healer_spec.rb
-- spec/networking/net_http_spec.rb
+- spec/networking/http_spec.rb
 - spec/networking/pool_spec.rb
 - spec/networking/selenium_spec.rb
+- spec/networking/strategy.rb
 - spec/page_spec.rb
 - spec/parsing/json_spec.rb
 - spec/parsing/xml_spec.rb
@@ -469,18 +510,18 @@ files:
 - spec/redis/counter_spec.rb
 - spec/redis/pool_spec.rb
 - spec/redis/version_spec.rb
-- spec/routing/custom_matcher_spec.rb
 - spec/routing/dsl_spec.rb
-- spec/routing/host_matcher_spec.rb
 - spec/routing/integration_spec.rb
+- spec/routing/matchers/custom_spec.rb
+- spec/routing/matchers/host_spec.rb
+- spec/routing/matchers/path_spec.rb
+- spec/routing/matchers/query_spec.rb
+- spec/routing/matchers/scheme_spec.rb
+- spec/routing/matchers/suffix_spec.rb
+- spec/routing/matchers/uri_spec.rb
 - spec/routing/path_finder_spec.rb
-- spec/routing/path_matcher_spec.rb
-- spec/routing/query_matcher_spec.rb
 - spec/routing/root_route_spec.rb
 - spec/routing/route_spec.rb
-- spec/routing/scheme_matcher_spec.rb
-- spec/routing/suffix_matcher_spec.rb
-- spec/routing/uri_matcher_spec.rb
 - spec/spec_helpers.rb
 - spec/stringify_spec.rb
 - spec/support/static/finders.html

data/lib/wayfarer/config.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  # rubocop:disable Lint/ConstantDefinitionInBlock
-  Config = Struct.new(:adapter,
-                      :adapter_pool_size,
-                      :adapter_pool_timeout,
-                      :ferrum_options,
-                      :selenium_argv,
-                      :redis_url,
-                      :redis_factory,
-                      :http_headers) do
-    DEFAULTS = {
-      adapter: :net_http,
-      adapter_pool_size: 3,
-      adapter_pool_timeout: ::ConnectionPool::DEFAULTS[:timeout],
-      ferrum_options: {},
-      selenium_argv: %i[chrome],
-      redis_url: "redis://localhost:6379",
-      redis_factory: -> { ::Redis.new(url: Wayfarer.config.redis_url) },
-      http_headers: {}
-    }.freeze
-
-    def self.default
-      new(*DEFAULTS.values)
-    end
-
-    # rubocop:disable Metrics/AbcSize
-    def self.from_environment(env = ENV)
-      default.tap do |config|
-        config.adapter = env["WAYFARER_ADAPTER"].to_sym if env.key?("WAYFARER_ADAPTER")
-        config.adapter_pool_size = cast(env["WAYFARER_POOL_SIZE"]) if env.key?("WAYFARER_POOL_SIZE")
-        config.adapter_pool_timeout = cast(env["WAYFARER_POOL_TIMEOUT"]) if env.key?("WAYFARER_POOL_TIMEOUT")
-        config.ferrum_options = parse_hash(env["WAYFARER_FERRUM_OPTIONS"]) if env.key?("WAYFARER_FERRUM_OPTIONS")
-        config.selenium_argv = parse_array(env["WAYFARER_SELENIUM_ARGV"]) if env.key?("WAYFARER_SELENIUM_ARGV")
-        config.redis_url = env["WAYFARER_REDIS_URL"] if env.key?("WAYFARER_REDIS_URL")
-        config.http_headers = parse_hash(env["WAYFARER_HTTP_HEADERS"]) if env.key?("WAYFARER_HTTP_HEADERS")
-      end
-    end
-    # rubocop:enable Metrics/AbcSize
-
-    def self.parse_hash(str)
-      parse_array(str).reduce({}) do |acc, pair|
-        k, v = pair.split(":", 2)
-        next acc unless k && v
-
-        acc.merge({ k.to_sym => cast(v) })
-      end
-    end
-
-    def self.parse_array(str)
-      str.split(",").map(&:strip)
-    end
-
-    def self.cast(str)
-      return true if str == "true"
-      return false if str == "false"
-
-      begin
-        Integer(str)
-      rescue StandardError
-        str
-      end
-    end
-  end
-  # rubocop:enable Lint/ConstantDefinitionInBlock
-end

data/lib/wayfarer/networking/healer.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-# An adapter that renews another adapter on certain errors.
-# This wrapper is required because connection_pool assumes
-# all connections are self-healing.
-module Wayfarer
-  module Networking
-    Healer = Struct.new(:adapter) do
-      extend Forwardable
-
-      delegate %i[body browser capybara free] => :adapter
-
-      def fetch(url)
-        adapter.fetch(url)
-      rescue *adapter.class.renew_on => e
-        adapter.renew
-        raise e
-      end
-    end
-  end
-end

data/lib/wayfarer/networking/net_http.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Networking
-    class NetHTTP
-      include Singleton
-
-      def self.renew_on
-        []
-      end
-
-      attr_reader :conn,
-                  :body
-
-      def initialize
-        @conn = Net::HTTP::Persistent.new("wayfarer-#{SecureRandom.uuid}")
-
-        Wayfarer.config.http_headers.each do |key, val|
-          conn.override_headers[key] = val
-        end
-      end
-
-      def renew; end
-
-      def browser; end
-
-      def capybara; end
-
-      def fetch(url)
-        res = conn.request(URI(url))
-
-        return Result::Redirect.new(url, res["location"]) if res.is_a?(Net::HTTPRedirection)
-
-        @body = res.body
-        page = Wayfarer::Page.new(url: url,
-                                  status_code: res.code.to_i,
-                                  body: body,
-                                  headers: res.to_hash)
-
-        Result::Success.new(page)
-      end
-
-      def live(current_page)
-        current_page
-      end
-
-      def free
-        conn.shutdown
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/custom_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class CustomMatcher
-      attr_reader :delegate
-
-      def initialize(delegate = proc)
-        @delegate = delegate
-      end
-
-      def match(url)
-        !!delegate.call(url)
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/host_matcher.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class HostMatcher
-      attr_reader :host
-
-      def initialize(host)
-        @host = host
-      end
-
-      # rubocop:disable Style/CaseEquality
-      def match(url)
-        @host === url.host
-      end
-      # rubocop:enable Style/CaseEquality
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/path_matcher.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class PathMatcher
-      attr_reader :path,
-                  :route,
-                  :peeking,
-                  :matcher
-
-      def initialize(path, route)
-        @path = path
-        @route = route
-        @peeking = false
-        @matcher = Mustermann.new(path, type: "sinatra")
-      end
-
-      def match(url)
-        route.accept(self)
-
-        # If the route's branch contains other path matchers in child routes,
-        # match the beginning of the path (peeking), instead of the whole path.
-        !!(if peeking
-             matcher.peek(url.path)
-           else
-             matcher.match(url.path)
-           end)
-      end
-
-      def params(url)
-        return {} unless match(url)
-
-        matcher.params(url.path) || {}
-      end
-
-      def visit(route)
-        return true if route == self.route
-
-        return unless route.matcher.is_a?(self.class)
-
-        @peeking = true
-        false
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/query_matcher.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class QueryMatcher
-      attr_reader :fields
-
-      def initialize(fields)
-        @fields = fields
-      end
-
-      def match(url)
-        query = url.query
-
-        # CGI::parse throws a NoMethodError if the query is an empty string
-        return false if query.nil? || query.empty?
-
-        CGI.parse(query).none? { |field, vals| violates?(field, vals) }
-      end
-
-      def params(url)
-        return {} unless match(url)
-
-        CGI.parse(url.query)
-           .select { |(k, _)| fields.keys.include?(k.to_sym) }
-           .transform_values(&:last)
-      end
-
-    private
-
-      # rubocop:disable Lint/AssignmentInCondition
-      def violates?(field, vals)
-        return false unless constraint = fields[field.to_sym]
-
-        violates_constraint?(constraint, vals)
-      end
-      # rubocop:enable Lint/AssignmentInCondition
-
-      def violates_constraint?(constraint, vals)
-        case constraint
-        when String  then violates_string?(constraint, vals)
-        when Integer then violates_integer?(constraint, vals)
-        when Regexp  then violates_regexp?(constraint, vals)
-        when Range   then violates_range?(constraint, vals)
-        end
-      end
-
-      def violates_string?(str, vals)
-        vals.none? { |val| str == val }
-      end
-
-      def violates_integer?(int, vals)
-        vals.none? { |val| int == Integer(val) }
-      rescue ArgumentError
-        true
-      end
-
-      def violates_regexp?(regexp, vals)
-        vals.none? { |val| regexp.match(val) }
-      end
-
-      def violates_range?(range, vals)
-        vals.none? { |val| range.include?(val.to_i) }
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/scheme_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class SchemeMatcher
-      attr_reader :scheme
-
-      def initialize(scheme)
-        @scheme = scheme
-      end
-
-      def match(url)
-        url.scheme == scheme.to_s
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/suffix_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class SuffixMatcher
-      attr_reader :suffix
-
-      def initialize(suffix)
-        @suffix = suffix
-      end
-
-      def match(url)
-        url.path.end_with?(suffix)
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/url_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class URLMatcher
-      attr_reader :url
-
-      def initialize(url)
-        @url = url
-      end
-
-      def match(url)
-        url == URI(@url)
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end