wayfarer 0.4.5 → 0.4.7

data/spec/networking/pool_spec.rb

@@ -17,7 +17,7 @@ describe Wayfarer::Networking::Pool do
     end
 
     context "when using Ferrum", ferrum: true do
-      before { Wayfarer.config.network.agent = :ferrum }
+      before { Wayfarer.config[:network][:agent] = :ferrum }
 
       it "yields Ferrum" do
         pool.with do |context|
@@ -27,7 +27,7 @@ describe Wayfarer::Networking::Pool do
     end
 
     context "when using Selenium", selenium: true do
-      before { Wayfarer.config.network.agent = :selenium }
+      before { Wayfarer.config[:network][:agent] = :selenium }
 
       it "yields Selenium" do
         pool.with do |context|
@@ -38,8 +38,8 @@ describe Wayfarer::Networking::Pool do
 
     context "when using Capybara", ferrum: true do
       before do
-        Wayfarer.config.network.agent = :capybara
-        Wayfarer.config.capybara.driver = :cuprite
+        Wayfarer.config[:network][:agent] = :capybara
+        Wayfarer.config[:capybara][:driver] = :cuprite
       end
 
       it "yields Capybara" do
@@ -55,7 +55,7 @@ describe Wayfarer::Networking::Pool do
 
     before do
       pool.class.registry[:foobar] = double(new: strategy)
-      Wayfarer.config.network.agent = :foobar
+      Wayfarer.config[:network][:agent] = :foobar
     end
 
     it "destroys the strategy" do

data/spec/networking/strategy.rb

@@ -76,7 +76,7 @@ RSpec.shared_examples "Network strategy" do |options|
 
       describe "HTTP headers" do
         before do
-          Wayfarer.config.network.http_headers = { "User-Agent" => "Foobar" }
+          Wayfarer.config[:network][:http_headers] = { "User-Agent" => "Foobar" }
           context.renew
         end
 
@@ -158,7 +158,7 @@ RSpec.shared_examples "Network strategy" do |options|
 
     describe "Client timeout" do
       before do
-        Wayfarer.config.selenium.client_timeout = 42
+        Wayfarer.config[:selenium][:client_timeout] = 42
         context.renew
       end
 

data/spec/page_spec.rb

@@ -3,45 +3,67 @@
 require "spec_helpers"
 
 describe Wayfarer::Page do
+  describe "#mime_type" do
+    let(:headers) { { "Content-Type" => "text/html; charset=UTF-8" } }
+
+    subject { build(:page, headers: headers).mime_type }
+
+    it { is_expected.to eq("text/html") }
+  end
+
   describe "#doc" do
     context "when Content-Type is absent" do
-      subject(:page) { build(:page, :html, headers: {}) }
+      subject { build(:page, :html, headers: {}).doc }
 
-      it "returns a Nokogiri::HTML::Document" do
-        expect(page.doc).to be_a(Nokogiri::HTML::Document)
-      end
+      it { is_expected.to be(nil) }
     end
 
     context "when Content-Type is HTML" do
-      subject(:page) { build(:page, :html) }
+      subject(:page) { build(:page, :html).doc }
 
-      it "returns a Nokogiri::HTML::Document" do
-        expect(page.doc).to be_a(Nokogiri::HTML::Document)
-      end
+      it { is_expected.to be_a(Nokogiri::HTML::Document) }
     end
 
     context "when Content-Type is XML" do
-      subject(:page) { build(:page, :xml) }
+      subject(:page) { build(:page, :xml).doc }
 
-      it "returns a Nokogiri::XML::Document" do
-        expect(page.doc).to be_a(Nokogiri::XML::Document)
-      end
+      it { is_expected.to be_a(Nokogiri::XML::Document) }
     end
 
     context "when Content-Type is JSON" do
-      subject(:page) { build(:page, :json) }
+      subject(:page) { build(:page, :json).doc }
 
-      it "returns a Hash" do
-        expect(page.doc).to be_a(Hash)
-      end
+      it { is_expected.to be_a(Hash) }
+    end
+
+    context "with Content-Type parameters" do
+      subject(:page) { build(:page, headers: { "content-type" => "text/html; charset=UTF-8" }).doc }
+
+      it { is_expected.to be_a(Nokogiri::HTML::Document) }
+    end
+
+    context "when Content-Type is unsupported" do
+      subject(:page) { build(:page, headers: { "Content-Type" => "foo/bar" }).doc }
+
+      it { is_expected.to be_nil }
     end
   end
 
-  describe "#meta" do
-    subject(:page) { build(:page, :html) }
+  describe "#headers" do
+    subject(:headers) { build(:page, headers: { "Content-Type" => "text/html" }).headers }
+
+    it "downcases keys" do
+      expect(headers).to eq("content-type" => "text/html")
+    end
 
-    it "returns a MetaInspector document" do
-      expect(page.meta).to be_a(MetaInspector::Document)
+    it "is case-sensitive" do
+      expect(headers["Content-Type"]).to be_nil
     end
   end
+
+  describe "#meta" do
+    subject(:page) { build(:page).meta }
+
+    it { is_expected.to be_a(MetaInspector::Document) }
+  end
 end

data/spec/parsing/xml_spec.rb

@@ -2,24 +2,23 @@
 
 require "spec_helpers"
 
-describe Wayfarer::Parsing::XML do
-  subject(:parser) { Wayfarer::Parsing::XML }
+describe Wayfarer::Parsing::XML, ".parse" do
+  let(:xml) { "<span>Foobar</span>" }
+  subject(:doc) { Wayfarer::Parsing::XML.parse(xml, variant) }
 
-  describe ".parse_html" do
-    it "returns a Nokogiri::HTML::Document" do
-      html_str = "<span>Foobar</span>"
-      doc = parser.parse_html(html_str)
+  context "HTML" do
+    let(:variant) { :html }
 
-      expect(doc).to be_a Nokogiri::HTML::Document
+    specify do
+      expect(doc).to be_a(Nokogiri::HTML::Document)
     end
   end
 
-  describe ".parse_xml" do
-    it "returns a Nokogiri::XML::Document" do
-      xml_str = "<barqux>Foobar</barqux>"
-      doc = parser.parse_xml(xml_str)
+  context "XML" do
+    let(:variant) { :xml }
 
-      expect(doc).to be_a Nokogiri::XML::Document
+    specify do
+      expect(doc).to be_a(Nokogiri::XML::Document)
     end
   end
 end

data/spec/redis/barrier_spec.rb

@@ -3,8 +3,8 @@
 require "spec_helpers"
 
 describe Wayfarer::Redis::Barrier, redis: true do
-  let(:url) { "https://example.com" }
-  subject(:barrier) { Wayfarer::Redis::Barrier.new("batch") }
+  let(:task) { build(:task, :redis_pool) }
+  subject(:barrier) { Wayfarer::Redis::Barrier.new(task) }
 
   describe "#redis_key" do
     it "returns the expected Redis key" do
@@ -14,65 +14,25 @@ describe Wayfarer::Redis::Barrier, redis: true do
 
   describe "#reset!" do
     it "resets seen URLs" do
-      barrier.seen?(url)
+      barrier.check!(task.url)
       barrier.reset!
-      expect(barrier.seen?(url)).to be false
+      expect(barrier.check!(task.url)).to be(false)
     end
   end
 
-  describe "#seen?" do
+  describe "#check!" do
     context "with seen URL" do
-      before { barrier.seen?(url) }
+      before { barrier.check!(task.url) }
 
       it "returns true" do
-        expect(barrier.seen?(url)).to be(true)
+        expect(barrier.check!(task.url)).to be(true)
       end
     end
 
     context "with unseen URL" do
       it "returns false" do
-        expect(barrier.seen?(url)).to be(false)
+        expect(barrier.check!(task.url)).to be(false)
       end
     end
   end
-
-  describe "#peek" do
-    let(:seen_urls) { %w[https://yahoo.com https://google.com] }
-    let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }
-
-    before do
-      allow(Wayfarer::Redis::Version).to receive(:determine).and_return(redis_version)
-
-      seen_urls.each do |url|
-        barrier.seen?(url)
-      end
-    end
-
-    context "Redis >= 6.2.0" do
-      let(:redis_version) { [6, 2, 5] }
-
-      it "returns membership" do
-        urls = [*seen_urls, *unseen_urls]
-        expect(barrier.peek(urls)).to eq([true, true, false, false])
-      end
-    end
-
-    context "Redis < 6.2.0" do
-      let(:redis_version) { [2, 6, 10] }
-
-      it "returns membership" do
-        urls = [*seen_urls, *unseen_urls]
-        expect(barrier.peek(urls)).to eq([true, true, false, false])
-      end
-    end
-  end
-
-  describe "#unsee" do
-    it "removes a seen URL" do
-      barrier.seen?(url)
-      expect {
-        barrier.unsee(url)
-      }.to change { barrier.seen?(url) }.to(false)
-    end
-  end
 end

data/spec/redis/counter_spec.rb

@@ -3,7 +3,9 @@
 require "spec_helpers"
 
 describe Wayfarer::Redis::Counter, redis: true do
-  subject(:counter) { Wayfarer::Redis::Counter.new("batch") }
+  let(:callback) { -> {} }
+  let(:task) { build(:task, :redis_pool) }
+  subject(:counter) { Wayfarer::Redis::Counter.new(task, &callback) }
 
   describe "#redis_key" do
     it "returns the expected Redis key" do
@@ -21,6 +23,16 @@ describe Wayfarer::Redis::Counter, redis: true do
     it "decrements" do
       expect { counter.decrement }.to change { counter.value }.by(-1)
     end
+
+    context "when reaching zero" do
+      let(:callback) { -> { @callback_called = true } }
+
+      before { counter.increment }
+
+      it "runs callback" do
+        expect { counter.decrement }.to change { @callback_called }.to(true).from(nil)
+      end
+    end
   end
 
   describe "#reset!" do

data/spec/redis/pool_spec.rb

@@ -8,7 +8,7 @@ describe Wayfarer::Redis::Pool, redis: true do
 
   describe "::with" do
     before do
-      Wayfarer.config.redis.factory = ->(_) { conn }
+      Wayfarer.config[:redis][:factory] = ->(_) { conn }
     end
 
     it "instantiates the factory and yields" do

data/spec/spec_helpers.rb

@@ -33,12 +33,12 @@ module SpecHelpers
     FileUtils.mkdir_p(Pathname.new(path).dirname)
     IO.write(path, contents)
   end
+end
 
-  # ActiveJob test helpers swallow errors
-  Base = Class.new(Wayfarer::Base) do
-    rescue_from(StandardError) do |e|
-      raise e
-    end
+module ActiveJobTestHelper
+  # perform_enqueued_jobs calls assert_equal
+  def assert_equal(expected, actual, _)
+    expect(actual).to eq(expected)
   end
 end
 
@@ -46,9 +46,13 @@ RSpec.configure do |config|
   config.include(FactoryBot::Syntax::Methods)
   config.include(ActiveJob::TestHelper)
   config.include(SpecHelpers)
+  config.include(ActiveJobTestHelper)
 
   config.before(:suite) do
+    Wayfarer::Logging.logger = Logger.new($stdout)
+
     FactoryBot.find_definitions
+    FactoryBot::SyntaxRunner.include SpecHelpers
 
     mutex = Mutex.new
     cvar = ConditionVariable.new
@@ -67,10 +71,8 @@ RSpec.configure do |config|
   end
 
   config.before do
-    Wayfarer.config = Wayfarer::Config::Root.new
-    Wayfarer.config.redis.url = redis_host
-
-    ::Redis.new(url: redis_host).flushall
+    Wayfarer.config = Wayfarer::DEFAULT_CONFIG.clone
+    Wayfarer.config[:redis][:url] = redis_host
 
     ActiveJob::Base.queue_adapter = :test
     ActiveJob::Base.logger = Logger.new(nil)
@@ -81,17 +83,21 @@ RSpec.configure do |config|
       # For unknown reasons, Wayfarer::Networking::Ferrum#renew can take a very
       # long time to instantiate a new ::Ferrum::Browser and requires the 60s
       # timeout. TODO: Figure out what's going on
-      Wayfarer.config.ferrum.options = { url: "http://chrome:3000", timeout: 120 }
-      Wayfarer.config.selenium.driver = :firefox
-      Wayfarer.config.selenium.options = { url: "http://firefox:4444" }
-      Wayfarer.config.selenium.client_timeout = 10
+      Wayfarer.config[:ferrum][:options] = { url: "http://chrome:3000", timeout: 120 }
+      Wayfarer.config[:selenium][:driver] = :firefox
+      Wayfarer.config[:selenium][:options] = { url: "http://firefox:4444" }
+      Wayfarer.config[:selenium][:client_timeout] = 10
     end
 
     # TODO: Undo side-effect
     Capybara.register_driver(:cuprite) do |app|
-      Capybara::Cuprite::Driver.new(app, Wayfarer.config.ferrum.options)
+      Capybara::Cuprite::Driver.new(app, Wayfarer.config[:ferrum][:options])
     end
-    Wayfarer.config.capybara.driver = :cuprite
+    Wayfarer.config[:capybara][:driver] = :cuprite
+  end
+
+  config.before(:each, redis: true) do
+    ::Redis.new(url: redis_host).flushall
   end
 
   config.around(cli: true) do |example|
@@ -99,7 +105,12 @@ RSpec.configure do |config|
     tmp = Dir.mktmpdir
     Dir.chdir(tmp)
     example.run
+  ensure
     Dir.chdir(origin)
-    FileUtils.rm_r(tmp)
+    # FileUtils.rm_r(tmp)
+  end
+
+  config.before(:each, cli: true) do
+    Object.send(:remove_const, :DummyJob) if defined?(DummyJob)
   end
 end

data/spec/support/test_app.rb

@@ -12,6 +12,14 @@ class TestApp < Sinatra::Base
     request.env[params[:header]]
   end
 
+  get "/response_header/:key/:val" do
+    headers params[:key] => params[:val]
+  end
+
+  get "/response_header/Content-Type/*" do
+    headers "Content-Type" => params[:splat].first
+  end
+
   get "/hello_world" do
     headers "hello" => "world"
     "Hello world!"

data/spec/task_spec.rb

@@ -5,30 +5,9 @@ require "spec_helpers"
 describe Wayfarer::Task do
   subject(:task) { build(:task) }
 
-  describe "#metadata" do
-    it "returns an OpenStruct" do
-      expect(task.metadata).to be_an(OpenStruct)
-    end
-  end
-
-  describe "#barrier" do
-    it "returns its batch barrier" do
-      expect(task.barrier).to be_a(Wayfarer::Redis::Barrier)
-      expect(task.barrier.batch).to eq(task.batch)
-    end
-  end
-
-  describe "#counter" do
-    it "returns its batch counter" do
-      expect(task.counter).to be_a(Wayfarer::Redis::Counter)
-      expect(task.counter.batch).to eq(task.batch)
-    end
-  end
-
-  describe "#gc" do
-    it "returns a garbage collector" do
-      expect(task.gc).to be_a(Wayfarer::GC)
-      expect(task.gc.task).to be(task)
+  describe "#[], #[]=" do
+    specify do
+      expect { task[:foo] = "bar" }.to change { task[:foo] }.from(nil).to("bar")
     end
   end
 end

data/spec/wayfarer_spec.rb

@@ -8,6 +8,6 @@ describe Wayfarer do
   end
 
   describe "::config" do
-    it("returns a config") { expect(Wayfarer.config).to be_a(Wayfarer::Config::Root) }
+    specify { expect(Wayfarer.config).to be_a(Hash) }
   end
 end

data/wayfarer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = "wayfarer"
-  s.version = "0.4.5"
+  s.version = "0.4.7"
   s.license = "MIT"
 
   s.homepage = "http://github.com/bauerd/wayfarer"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = "~> 2.7"
 
-  s.add_runtime_dependency "activejob", ">= 6.0"
+  s.add_runtime_dependency "activejob", ">= 7.1"
   s.add_runtime_dependency "addressable", "~> 2.8"
   s.add_runtime_dependency "capybara", "~> 3.0"
   s.add_runtime_dependency "connection_pool", "~> 2.2"
@@ -30,14 +30,15 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "ferrum", "~> 0.9"
   s.add_runtime_dependency "metainspector", "~> 5.0"
   s.add_runtime_dependency "mime-types", "~> 3.0"
-  s.add_runtime_dependency "mustermann", "~> 1.1"
   s.add_runtime_dependency "mock_redis", "~> 0.29"
+  s.add_runtime_dependency "mustermann", "~> 1.1"
   s.add_runtime_dependency "net-http-persistent", "~> 3.0"
   s.add_runtime_dependency "nokogiri", "~> 1.11"
   s.add_runtime_dependency "normalize_url", "~> 0.0.6"
   s.add_runtime_dependency "redis", "~> 4.4", "< 4.5" # redis 4.5.x broke SMISMEMBER
   s.add_runtime_dependency "selenium-webdriver", "~> 3.4"
   s.add_runtime_dependency "thor", "~> 1.0"
+  s.add_runtime_dependency "zeitwerk", "~> 2.4"
 
   s.add_development_dependency "cuprite", "~> 0.13"
   s.add_development_dependency "factory_bot", "~> 6.0"