wayfarer 0.4.0 → 0.4.4

data/docs/guides/browser_automation/capybara.md

@@ -1,3 +0,0 @@
-# Capybara
-
-TODO

data/lib/wayfarer/config.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  # rubocop:disable Lint/ConstantDefinitionInBlock
-  Config = Struct.new(:adapter,
-                      :adapter_pool_size,
-                      :adapter_pool_timeout,
-                      :ferrum_options,
-                      :selenium_argv,
-                      :redis_url,
-                      :redis_factory,
-                      :http_headers) do
-    DEFAULTS = {
-      adapter: :net_http,
-      adapter_pool_size: 3,
-      adapter_pool_timeout: ::ConnectionPool::DEFAULTS[:timeout],
-      ferrum_options: {},
-      selenium_argv: %i[chrome],
-      redis_url: "redis://localhost:6379",
-      redis_factory: -> { ::Redis.new(url: Wayfarer.config.redis_url) },
-      http_headers: {}
-    }.freeze
-
-    def self.default
-      new(*DEFAULTS.values)
-    end
-
-    # rubocop:disable Metrics/AbcSize
-    def self.from_environment(env = ENV)
-      default.tap do |config|
-        config.adapter = env["WAYFARER_ADAPTER"].to_sym if env.key?("WAYFARER_ADAPTER")
-        config.adapter_pool_size = cast(env["WAYFARER_POOL_SIZE"]) if env.key?("WAYFARER_POOL_SIZE")
-        config.adapter_pool_timeout = cast(env["WAYFARER_POOL_TIMEOUT"]) if env.key?("WAYFARER_POOL_TIMEOUT")
-        config.ferrum_options = parse_hash(env["WAYFARER_FERRUM_OPTIONS"]) if env.key?("WAYFARER_FERRUM_OPTIONS")
-        config.selenium_argv = parse_array(env["WAYFARER_SELENIUM_ARGV"]) if env.key?("WAYFARER_SELENIUM_ARGV")
-        config.redis_url = env["WAYFARER_REDIS_URL"] if env.key?("WAYFARER_REDIS_URL")
-        config.http_headers = parse_hash(env["WAYFARER_HTTP_HEADERS"]) if env.key?("WAYFARER_HTTP_HEADERS")
-      end
-    end
-    # rubocop:enable Metrics/AbcSize
-
-    def self.parse_hash(str)
-      parse_array(str).reduce({}) do |acc, pair|
-        k, v = pair.split(":", 2)
-        next acc unless k && v
-
-        acc.merge({ k.to_sym => cast(v) })
-      end
-    end
-
-    def self.parse_array(str)
-      str.split(",").map(&:strip)
-    end
-
-    def self.cast(str)
-      return true if str == "true"
-      return false if str == "false"
-
-      begin
-        Integer(str)
-      rescue StandardError
-        str
-      end
-    end
-  end
-  # rubocop:enable Lint/ConstantDefinitionInBlock
-end

data/lib/wayfarer/middleware/worker.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Middleware
-    module Worker
-      def self.included(base)
-        base.include(Wayfarer::Redis::Connection)
-        base.include(Wayfarer::Middleware::Stage::API)
-        base.include(InstanceMethods)
-        base.extend(ClassMethods)
-      end
-
-      module ClassMethods
-        def route
-          @route ||= Wayfarer::Routing::RootRoute.new.tap do |route|
-            Docile.dsl_eval(route) { yield if block_given? }
-          end
-        end
-      end
-
-      module InstanceMethods
-        extend Forwardable
-
-        attr_accessor :task
-
-        delegate %i[params adapter] => "task.metadata"
-        delegate %i[browser capybara] => :adapter
-
-        def call(task)
-          self.task = task
-          public_send(task.metadata.action)
-          yield if block_given?
-        end
-
-        def chain
-          Wayfarer::Middleware::Chain.new([*Wayfarer.core_middleware, self])
-        end
-
-        def page(live: false)
-          return task.metadata.page unless live
-
-          task.metadata.page = adapter.live(task.metadata.page)
-        end
-      end
-    end
-  end
-end

data/lib/wayfarer/networking/healer.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-# An adapter that renews another adapter on certain errors.
-# This wrapper is required because connection_pool assumes
-# all connections are self-healing.
-module Wayfarer
-  module Networking
-    Healer = Struct.new(:adapter) do
-      extend Forwardable
-
-      delegate %i[body browser capybara free] => :adapter
-
-      def fetch(url)
-        adapter.fetch(url)
-      rescue *adapter.class.renew_on => e
-        adapter.renew
-        raise e
-      end
-    end
-  end
-end

data/lib/wayfarer/networking/net_http.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Networking
-    class NetHTTP
-      include Singleton
-
-      def self.renew_on
-        []
-      end
-
-      attr_reader :conn,
-                  :body
-
-      def initialize
-        @conn = Net::HTTP::Persistent.new("wayfarer-#{SecureRandom.uuid}")
-
-        Wayfarer.config.http_headers.each do |key, val|
-          conn.override_headers[key] = val
-        end
-      end
-
-      def renew; end
-
-      def browser; end
-
-      def capybara; end
-
-      def fetch(url)
-        res = conn.request(URI(url))
-
-        return Result::Redirect.new(url, res["location"]) if res.is_a?(Net::HTTPRedirection)
-
-        @body = res.body
-        page = Wayfarer::Page.new(url: url,
-                                  status_code: res.code.to_i,
-                                  body: body,
-                                  headers: res.to_hash)
-
-        Result::Success.new(page)
-      end
-
-      def live(current_page)
-        current_page
-      end
-
-      def free
-        conn.shutdown
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/custom_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class CustomMatcher
-      attr_reader :delegate
-
-      def initialize(delegate = proc)
-        @delegate = delegate
-      end
-
-      def match(url)
-        !!delegate.call(url)
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/host_matcher.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class HostMatcher
-      attr_reader :host
-
-      def initialize(host)
-        @host = host
-      end
-
-      # rubocop:disable Style/CaseEquality
-      def match(url)
-        @host === url.host
-      end
-      # rubocop:enable Style/CaseEquality
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/path_matcher.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class PathMatcher
-      attr_reader :path,
-                  :route,
-                  :peeking,
-                  :matcher
-
-      def initialize(path, route)
-        @path = path
-        @route = route
-        @peeking = false
-        @matcher = Mustermann.new(path, type: "sinatra")
-      end
-
-      def match(url)
-        route.accept(self)
-
-        # If the route's branch contains other path matchers in child routes,
-        # match the beginning of the path (peeking), instead of the whole path.
-        !!(if peeking
-             matcher.peek(url.path)
-           else
-             matcher.match(url.path)
-           end)
-      end
-
-      def params(url)
-        return {} unless match(url)
-
-        matcher.params(url.path) || {}
-      end
-
-      def visit(route)
-        return true if route == self.route
-
-        return unless route.matcher.is_a?(self.class)
-
-        @peeking = true
-        false
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/query_matcher.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class QueryMatcher
-      attr_reader :fields
-
-      def initialize(fields)
-        @fields = fields
-      end
-
-      def match(url)
-        query = url.query
-
-        # CGI::parse throws a NoMethodError if the query is an empty string
-        return false if query.nil? || query.empty?
-
-        CGI.parse(query).none? { |field, vals| violates?(field, vals) }
-      end
-
-      def params(url)
-        return {} unless match(url)
-
-        CGI.parse(url.query)
-           .select { |(k, _)| fields.keys.include?(k.to_sym) }
-           .transform_values(&:last)
-      end
-
-    private
-
-      # rubocop:disable Lint/AssignmentInCondition
-      def violates?(field, vals)
-        return false unless constraint = fields[field.to_sym]
-
-        violates_constraint?(constraint, vals)
-      end
-      # rubocop:enable Lint/AssignmentInCondition
-
-      def violates_constraint?(constraint, vals)
-        case constraint
-        when String  then violates_string?(constraint, vals)
-        when Integer then violates_integer?(constraint, vals)
-        when Regexp  then violates_regexp?(constraint, vals)
-        when Range   then violates_range?(constraint, vals)
-        end
-      end
-
-      def violates_string?(str, vals)
-        vals.none? { |val| str == val }
-      end
-
-      def violates_integer?(int, vals)
-        vals.none? { |val| int == Integer(val) }
-      rescue ArgumentError
-        true
-      end
-
-      def violates_regexp?(regexp, vals)
-        vals.none? { |val| regexp.match(val) }
-      end
-
-      def violates_range?(range, vals)
-        vals.none? { |val| range.include?(val.to_i) }
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/scheme_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class SchemeMatcher
-      attr_reader :scheme
-
-      def initialize(scheme)
-        @scheme = scheme
-      end
-
-      def match(url)
-        url.scheme == scheme.to_s
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/suffix_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class SuffixMatcher
-      attr_reader :suffix
-
-      def initialize(suffix)
-        @suffix = suffix
-      end
-
-      def match(url)
-        url.path.end_with?(suffix)
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/url_matcher.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    class URLMatcher
-      attr_reader :url
-
-      def initialize(url)
-        @url = url
-      end
-
-      def match(url)
-        url == URI(@url)
-      end
-
-      def params(_)
-        {}
-      end
-    end
-  end
-end

data/spec/config_spec.rb

@@ -1,144 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config do
-  describe "::defaults" do
-    it "returns the default config" do
-      config = Wayfarer::Config.default
-      expect(config.adapter).to be(:net_http)
-      expect(config.adapter_pool_size).to be(3)
-      expect(config.adapter_pool_timeout).to eq(ConnectionPool::DEFAULTS[:timeout])
-      expect(config.ferrum_options).to eq({})
-      expect(config.selenium_argv).to eq(%i[chrome])
-      expect(config.redis_url).to eq("redis://localhost:6379")
-      expect(config.redis_factory).not_to be(nil)
-    end
-  end
-
-  describe "::from_environment" do
-    let(:env) { {} }
-    subject(:config) { Wayfarer::Config.from_environment(env) }
-
-    describe "#adapter" do
-      context "by default" do
-        it "returns :net_http" do
-          expect(config.adapter).to be(:net_http)
-        end
-      end
-
-      context "with env var set" do
-        before { env["WAYFARER_ADAPTER"] = "ferrum" }
-
-        it "symbolizes the env var" do
-          expect(config.adapter).to be(:ferrum)
-        end
-      end
-    end
-
-    describe "#adapter_pool_size" do
-      context "by default" do
-        it "returns the default config value" do
-          expect(config.adapter_pool_size).to be(3)
-        end
-      end
-
-      context "with env var set" do
-        before { env["WAYFARER_POOL_SIZE"] = "1234" }
-
-        it "parses the env var to a number" do
-          expect(config.adapter_pool_size).to be(1234)
-        end
-      end
-    end
-
-    describe "#adapter_pool_timeout" do
-      context "by default" do
-        it "returns the default config value" do
-          expect(config.adapter_pool_timeout).to be(ConnectionPool::DEFAULTS[:timeout])
-        end
-      end
-
-      context "with env var set" do
-        let(:env) { { "WAYFARER_POOL_TIMEOUT" => "1000" } }
-
-        it "parses the env var to a number" do
-          expect(config.adapter_pool_timeout).to be(1000)
-        end
-      end
-    end
-
-    describe "#ferrum_options" do
-      context "by default" do
-        it "is empty" do
-          expect(config.ferrum_options).to eq({})
-        end
-      end
-
-      context "with env var set" do
-        before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
-
-        it "parses the env var to a Hash" do
-          expect(config.ferrum_options).to eq(url: "http://chrome:3000", headless: false)
-        end
-      end
-    end
-
-    describe "#selenium_argv" do
-      context "by default" do
-        it "is [:chrome]" do
-          expect(config.selenium_argv).to eq(%i[chrome])
-        end
-      end
-
-      context "with env var set" do
-        before { env["WAYFARER_SELENIUM_ARGV"] = "firefox" }
-
-        it "parses the env var to an array" do
-          expect(config.selenium_argv).to eq(%w[firefox])
-        end
-      end
-    end
-
-    describe "#redis_url" do
-      context "by default" do
-        it "is redis://localhost:6379" do
-          expect(config.redis_url).to eq("redis://localhost:6379")
-        end
-      end
-
-      context "with env var set" do
-        before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
-
-        it "reads the env var" do
-          expect(config.redis_url).to eq("redis://redis:6379")
-        end
-      end
-    end
-
-    describe "#http_headers" do
-      context "by default" do
-        it "is {}" do
-          expect(config.http_headers).to eq({})
-        end
-      end
-
-      context "with env var set" do
-        before { env["WAYFARER_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
-
-        it "reads the env var" do
-          expect(config.http_headers).to eq("user-agent": "foo",
-                                            authorization: "bar")
-        end
-      end
-    end
-  end
-
-  describe "#redis_factory" do
-    let(:config) { Wayfarer::Config.default }
-
-    it "instantiates Redis clients" do
-      expect(config.redis_factory.call).to be_a(Redis)
-    end
-  end
-end

data/spec/factories/queue/chain.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-FactoryBot.define do
-  factory :chain, class: Wayfarer::Middleware::Chain do
-    middlewares { [] }
-
-    initialize_with do
-      new(middlewares)
-    end
-  end
-end

data/spec/middleware/worker_spec.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Worker do
-  let(:task) { build(:task) }
-
-  before { task.metadata.action = :index }
-
-  let(:job) do
-    Class.new(Wayfarer::Base) do
-      def index
-        :foobar
-      end
-    end
-  end
-
-  describe "::route" do
-    it "returns a route" do
-      expect(job.route).to be_a(Wayfarer::Routing::Route)
-    end
-  end
-
-  describe "#call" do
-    it "assigns the task" do
-      worker = job.new
-      expect { worker.call(task) }.to change { worker.task }.to(task)
-    end
-
-    it "calls the action method" do
-      worker = job.new
-      expect(worker).to receive(:index)
-      worker.call(task)
-    end
-
-    it "yields" do
-      expect { |spy| job.new.call(task, &spy) }.to yield_control
-    end
-  end
-
-  describe "#chain" do
-    it "returns its middleware chain" do
-      worker = job.new
-      expect(worker.chain.middlewares).to eq([*Wayfarer.core_middleware, worker])
-    end
-  end
-
-  describe "#redis" do
-    it "yields a Redis connection" do
-      job.new.redis do |conn|
-        expect(conn).to be_a(Redis)
-      end
-    end
-  end
-
-  describe "#page" do
-    let(:page) { Object.new }
-    let(:worker) { job.new }
-
-    before do
-      task.metadata.page = page
-      worker.task = task
-    end
-
-    it "returns the page" do
-      expect(worker.page).to be(page)
-    end
-
-    context "with live keyword" do
-      let(:adapter) { spy }
-      let(:body) { "foobar" }
-      let(:live) { Object.new }
-
-      before do
-        task.metadata.adapter = adapter
-        allow(adapter).to receive(:live).with(page).and_return(live)
-      end
-
-      it "returns a live page" do
-        expect(worker.page(live: true)).to be(live)
-      end
-
-      it "replaces the existing page" do
-        expect {
-          worker.page(live: true)
-        }.to change { task.metadata.page }.to(live)
-      end
-    end
-  end
-end