scruber 0.1.5 → 0.1.6

data/spec/scruber_spec.rb

@@ -0,0 +1,198 @@
+require "spec_helper"
+
+RSpec.describe Scruber do
+  before do
+    Scruber::Helpers::UserAgentRotator.configure do
+      clean
+      set_filter :all
+      add "Scruber 1.0", tags: [:robot, :scruber]
+      add "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36", tags: [:desktop, :chrome, :macos]
+    end
+  end
+
+  it "has a version number" do
+    expect(Scruber::VERSION).not_to be nil
+  end
+
+  describe "configurable" do
+    before do
+      Scruber.configure do |config|
+        config.fetcher_adapter = :typhoeus_fetcher
+      end
+    end
+    
+    it "returns :typhoeus_fetcher as fetcher" do
+      expect(Scruber.configuration.fetcher_adapter).to eq(:typhoeus_fetcher)
+    end
+  end
+
+  describe "#run" do
+    context "without args" do
+      it "should raise error" do
+        expect { Scruber.run { $title = scraper_name } }.to raise_error(Scruber::ArgumentError)
+      end
+
+      it "should set scraper name from ENV" do
+        ENV['SCRUBER_SCRAPER_NAME'] = 'sample'
+        Scruber.run do
+          $scraper_name = scraper_name
+        end
+        expect($scraper_name).to eq(:sample)
+      end
+    end
+
+    context "with args" do
+      it "should set scraper name from first arg" do
+        Scruber.run :sample1 do
+          $scraper_name = scraper_name
+        end
+        expect($scraper_name).to eq(:sample1)
+      end
+
+      it "should set scraper name from first arg, and options from second" do
+        Scruber.run :sample2, queue_adapter: :test do
+          $scraper_name = scraper_name
+          $opt = Scruber.configuration.queue_adapter
+        end
+        expect($scraper_name).to eq(:sample2)
+        expect($opt).to eq(:test)
+      end
+
+      it "options from first arg and scraper_name from ENV" do
+        ENV['SCRUBER_SCRAPER_NAME'] = 'sample'
+        Scruber.run queue_adapter: :test2 do
+          $scraper_name = scraper_name
+          $opt = Scruber.configuration.queue_adapter
+        end
+        expect($scraper_name).to eq(:sample)
+        expect($opt).to eq(:test2)
+      end
+
+      it "should raise error if passed only options without ENV" do
+        ENV['SCRUBER_SCRAPER_NAME'] = nil
+        expect { Scruber.run(queue_adapter: :test2) { $title = scraper_name } }.to raise_error(Scruber::ArgumentError)
+      end
+    end
+
+    it "simple example" do
+      stub_request(:get, "http://example.com").to_return(body: 'Example Domain')
+
+      Scruber.run :sample do
+        queue.add "http://example.com"
+        
+        parser :seed do |page|
+          $title = page.response_body
+        end
+      end
+      expect($title).to eq('Example Domain')
+    end
+
+    it "should return Nokogiri object" do
+      stub_request(:get, "http://example.com/contacts.html").to_return(body: '<div><a>Contacts</a></div>')
+
+      Scruber.run :sample do
+        queue.add "http://example.com/contacts.html"
+        
+        parser :seed, format: :html do |page, html|
+          $title = html.at('a').text
+        end
+      end
+      expect($title).to eq('Contacts')
+    end
+
+    context "complex example" do
+      it "should parse pages in 2 steps" do
+        stub_request(:get, "http://example.com/catalog").to_return(body: '<div><a href="/product1">Product 1</a><a href="/product2">Product 2</a><a href="/product3">Product 3</a></div>')
+        stub_request(:get, "http://example.com/product1").to_return(body: '<div><h1>Product 1</h1></div>')
+        stub_request(:get, "http://example.com/product2").to_return(body: '<div><h1>Product 2</h1></div>')
+        stub_request(:get, "http://example.com/product3").to_return(body: '<div><h1>Product 3</h1></div>')
+
+        $products = []
+        Scruber.run :sample do
+          get "http://example.com/catalog"
+          
+          parse :html do |page, doc|
+            doc.search('a').each do |a|
+              get_product URI.join(page.url, a.attr('href')).to_s
+            end
+          end
+
+          parse_product :html do |page,doc|
+            $products.push doc.at('h1').text
+          end
+        end
+        expect($products.sort).to eq((1..3).map{|i| "Product #{i}"}.sort)
+      end
+
+      it "should redownload page and increase retry" do
+        stub_request(:get, "http://example.com/").to_return(body: '<div>blocked</div>').times(2).then.to_return(body: '<div><h1>Product</h1></div>')
+
+        Scruber.run :sample do
+          get "http://example.com/"
+          
+          parse :html do |page, doc|
+            if page.response_body =~ /blocked/
+              page.redownload!
+            else
+              $title = doc.at('h1').text
+              $retry_count = page.retry_count
+            end
+          end
+        end
+        expect($title).to eq('Product')
+        expect($retry_count).to eq(2)
+      end
+    end
+
+    context "processing error examples" do
+      it "should process 500 error page" do
+        stub_request(:get, "http://example.com").to_return(body: '<div><h1>500</h1></div>', status: 500)
+
+        $error_title = nil
+        Scruber.run :sample do
+          get "http://example.com", max_retry_times: 1
+
+          parse :html do |page,doc|
+            $error_title = doc.at('h1').text
+          end
+
+          on_page_error do |page|
+            $error_title = page.response_body
+            page.processed!
+          end
+        end
+        expect($error_title).to eq('<div><h1>500</h1></div>')
+      end
+
+      it "should process 404 error page" do
+        stub_request(:get, "http://example.com").to_return(body: '<div><h1>404</h1></div>', status: 404)
+
+        $error_title = nil
+        Scruber.run :sample do
+          get "http://example.com", max_retry_times: 1
+
+          parse :html do |page,doc|
+            $error_title = doc.at('h1').text
+          end
+
+          on_page_error do |page|
+            $error_title = page.response_body
+            page.processed!
+          end
+        end
+        expect($error_title).to eq('<div><h1>404</h1></div>')
+      end
+    end
+  end
+
+  describe "#root" do
+    it "should return nil without APP_PATH defined" do
+      expect(Scruber.root).to eq(nil)
+    end
+
+    it "should return path object" do
+      APP_PATH='/tmp/a/b/'
+      expect(Scruber.root.to_s).to eq('/tmp')
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,36 @@
+require "bundler/setup"
+require "scruber"
+require 'webmock/rspec'
+
+Encoding.default_external = Encoding::UTF_8
+Encoding.default_internal = Encoding::UTF_8
+
+Dir[File.expand_path(File.dirname(__FILE__))+"/support/**/*.rb"].each { |f| require f }
+
+Scruber::Helpers::UserAgentRotator.configure do
+  set_filter :all
+  add "Scruber 1.0", tags: [:robot, :scruber]
+  add "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36", tags: [:desktop, :chrome, :macos]
+end
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+
+  # Use color in STDOUT
+  config.color = true
+
+  # Use color not only in STDOUT but also in pagers and files
+  config.tty = true
+
+  # Use the specified formatter
+  config.formatter = :progress # :documentation, :html, :textmate
+
+end

data/spec/support/queue/queue_adapter.rb

@@ -0,0 +1,171 @@
+require "spec_helper"
+
+RSpec.shared_examples "queue_adapter" do
+
+  it "should update page" do
+    queue.add "http://example.com"
+    page = queue.fetch_pending
+    page.url = "http://example.net"
+    page.save
+    page = queue.fetch_pending
+    expect(page.url).to eq("http://example.net")
+  end
+
+  it "should update page and fetch downloaded page" do
+    queue.add "http://example.com"
+    page = queue.fetch_pending
+    page.fetched_at = Time.now.to_i
+    page.save
+    pending_page = queue.fetch_pending
+    downloaded_page = queue.fetch_downloaded
+    expect(pending_page).to eq(nil)
+    expect(downloaded_page.url).to eq("http://example.com")
+  end
+
+  describe "processing errors page" do
+    it "should fetch error page" do
+      queue.add "http://example.com"
+      page = queue.fetch_pending
+      page.retry_count = 5
+      page.max_retry_times = 5
+      page.save
+      error_page = queue.fetch_error
+      expect(error_page).not_to eq(nil)
+      expect(error_page.id).to eq(page.id)
+    end
+
+    it "should return page to downloading" do
+      queue.add "http://example.com"
+      page = queue.fetch_pending
+      page.retry_count = 5
+      page.max_retry_times = 5
+      page.save
+      error_page = queue.fetch_error
+      error_page.redownload!(0)
+      pending_page = queue.fetch_pending
+      err_page = queue.fetch_error
+      d_page = queue.fetch_downloaded
+      expect(error_page.id).to eq(pending_page.id)
+      expect(err_page).to be_nil
+      expect(d_page).to be_nil
+    end
+
+    it "should delete page from queue" do
+      queue.add "http://example.com"
+      page = queue.fetch_pending
+      page.retry_count = 5
+      page.max_retry_times = 5
+      page.save
+      error_page = queue.fetch_error
+      error_page.delete
+      pending_page = queue.fetch_pending
+      err_page = queue.fetch_error
+      d_page = queue.fetch_downloaded
+      expect(pending_page).to be_nil
+      expect(err_page).to be_nil
+      expect(d_page).to be_nil
+    end
+
+    it "should process page" do
+      queue.add "http://example.com"
+      page = queue.fetch_pending
+      page.retry_count = 5
+      page.max_retry_times = 5
+      page.save
+      error_page = queue.fetch_error
+      error_page.processed!
+      pending_page = queue.fetch_pending
+      err_page = queue.fetch_error
+      d_page = queue.fetch_downloaded
+      expect(pending_page).to be_nil
+      expect(err_page).to be_nil
+      expect(d_page).to be_nil
+      queue.add "http://example.com"
+      pending_page = queue.fetch_pending
+      expect(pending_page).to be_nil
+    end
+  end
+
+  context "#add" do
+    it "queue page for downloading" do
+      queue.add "http://example.com"
+      expect(queue.size).to eq(1)
+    end
+
+    it "should not add the same page twice" do
+      queue.add "http://example.com"
+      expect(queue.size).to eq(1)
+      queue.add "http://example.com"
+      expect(queue.size).to eq(1)
+    end
+
+    it "should not add the same page twice even if page was processed" do
+      queue.add "http://example.com"
+      page = queue.fetch_pending
+      page.fetched_at = Time.now.to_i
+      page.save
+      downloaded_page = queue.fetch_downloaded
+      downloaded_page.processed!
+      queue.add "http://example.com"
+      page = queue.fetch_pending
+      expect(page).to eq(nil)
+    end
+  end
+
+  context "#save" do
+    it "should delete page" do
+      queue.add "http://example.abc"
+      page = queue.fetch_pending
+      page.fetched_at = Time.now.to_i
+      page.save
+      page.delete
+      page = queue.fetch_downloaded
+      
+      expect(page).to eq(nil)
+    end
+
+    it "should save additional arguments" do
+      queue.add "http://example.abc", id: 'abc', test_id: '1'
+      page = queue.find 'abc'
+      
+      expect(page.options[:test_id]).to eq('1')
+    end
+
+    it "should not override page" do
+      queue.add "http://example.abc", id: 'abc'
+      page = queue.find 'abc'
+      page.fetched_at = 1
+      page.save
+      page = queue.find 'abc'
+      expect(page.fetched_at).to eq(1)
+      queue.add "http://example.abc", id: 'abc'
+      page = queue.find 'abc'
+      expect(page.fetched_at).to eq(1)
+    end
+  end
+
+  context "#processed!" do
+    it "should update page and set processed_at" do
+      queue.add "http://example.com"
+      page = queue.fetch_pending
+      page.fetched_at = Time.now.to_i
+      page.save
+      downloaded_page = queue.fetch_downloaded
+      downloaded_page.processed!
+      downloaded_page2 = queue.fetch_downloaded
+      expect(downloaded_page2).to eq(nil)
+      expect(downloaded_page.processed_at).to be >= 0
+    end
+  end
+
+  describe "Page" do
+    let(:page_class){ described_class.const_get(:Page) }
+
+    it "should generate different ids for different urls" do
+      page1 = page_class.new queue, url: "http://example.com/product1"
+      page2 = page_class.new queue, url: "http://example.com/product2"
+      expect(page1.id).not_to be_blank
+      expect(page1.id).not_to eq(page2.id)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scruber
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Ivan Goncharov
@@ -269,6 +269,31 @@ files:
 - lib/scruber/queue_adapters/memory.rb
 - lib/scruber/version.rb
 - scruber.gemspec
+- spec/core/extensions/csv_output_spec.rb
+- spec/core/extensions/dict.csv
+- spec/core/extensions/log_spec.rb
+- spec/core/extensions/loop_spec.rb
+- spec/core/extensions/parser_aliases_spec.rb
+- spec/core/extensions/queue_aliases_spec.rb
+- spec/core/extensions/seed_spec.rb
+- spec/fetcher.rb
+- spec/helpers/dictionary_reader/dict.csv
+- spec/helpers/dictionary_reader/dict.xml
+- spec/helpers/dictionary_reader/dict_records.xml
+- spec/helpers/dictionary_reader/dictionary_reader_csv_spec.rb
+- spec/helpers/dictionary_reader/dictionary_reader_xml_spec.rb
+- spec/helpers/fetcher_agent_adapters/abstract_adapter_spec.rb
+- spec/helpers/fetcher_agent_adapters/memory_spec.rb
+- spec/helpers/proxy_rotator/proxy_rotator_proxy_spec.rb
+- spec/helpers/proxy_rotator/proxy_rotator_spec.rb
+- spec/helpers/user_agent_rotator/user_agent_rotator_spec.rb
+- spec/helpers/user_agent_rotator/user_agent_rotator_user_agent_spec.rb
+- spec/helpers/user_agent_rotator/user_agents.xml
+- spec/queue_adapter/memory_spec.rb
+- spec/queue_spec.rb
+- spec/scruber_spec.rb
+- spec/spec_helper.rb
+- spec/support/queue/queue_adapter.rb
 homepage: https://github.com/scruber/scruber
 licenses:
 - MIT