scruber 0.1.5 → 0.1.6

data/spec/helpers/fetcher_agent_adapters/abstract_adapter_spec.rb

@@ -0,0 +1,46 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::FetcherAgentAdapters::AbstractAdapter do
+
+  let(:cookie_jar_string) { "---\n- !ruby/object:HTTP::Cookie\n  name: feed_flow\n  value: top\n  domain: example.com\n  for_domain: false\n  path: \"/\"\n  secure: false\n  httponly: true\n  expires: \n  max_age: 26784000\n  created_at: #{Time.now.strftime('%Y-%m-%d')} 16:46:15.443984000 +03:00\n  accessed_at: #{Time.now.strftime('%Y-%m-%d')} 16:47:07.047296000 +03:00\n" }
+
+  describe "initialize" do
+    let(:agent) do
+      described_class.new id: 1,
+                          user_agent: 'Scruber',
+                          proxy_id: 1,
+                          headers: {'a' => 1},
+                          cookie_jar: cookie_jar_string,
+                          disable_proxy: true
+    end
+    
+    it "set values" do
+      expect(agent.id).to eq(1)
+      expect(agent.user_agent).to eq('Scruber')
+      expect(agent.proxy_id).to eq(1)
+      expect(agent.headers).to eq({'a' => 1})
+      expect(agent.cookie_jar).to eq(cookie_jar_string)
+      expect(agent.disable_proxy).to eq(true)
+    end
+
+    it "load cookies" do
+      expect(agent.cookie_for('http://example.com')).to eq('feed_flow=top')
+      expect(agent.cookie_for(URI('http://example.com'))).to eq('feed_flow=top')
+    end
+
+    it "serialize cookie" do
+      expect(agent.serialize_cookies).to eq(cookie_jar_string)
+    end
+
+    it "parse cookies from page" do
+      page = Scruber::QueueAdapters::AbstractAdapter::Page.new(nil, url: 'http://example.com', response_headers: {"Connection" => "keep-alive","Set-Cookie" => "__cfduid=dc8db498b1e419b7943052a69c8e9d1d01504311966; expires=Sun, 02-Sep-18 00:26:06 GMT; path=/; domain=example.com; HttpOnly"})
+      agent.parse_cookies_from_page!(page)
+      expect(agent.cookie_for('http://example.com')).to eq('__cfduid=dc8db498b1e419b7943052a69c8e9d1d01504311966; feed_flow=top')
+    end
+
+  end
+
+  it "should be accessible from scraper" do
+    expect { Scruber.run(:sample) { FetcherAgent } }.not_to raise_error
+  end
+end

data/spec/helpers/fetcher_agent_adapters/memory_spec.rb

@@ -0,0 +1,45 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::FetcherAgentAdapters::Memory do
+
+  let(:cookie_jar_string) { "---\n- !ruby/object:HTTP::Cookie\n  name: feed_flow\n  value: top\n  domain: example.com\n  for_domain: false\n  path: \"/\"\n  secure: false\n  httponly: true\n  expires: \n  max_age: 26784000\n  created_at: #{Time.now.strftime('%Y-%m-%d')} 16:46:15.443984000 +03:00\n  accessed_at: #{Time.now.strftime('%Y-%m-%d')} 16:47:07.047296000 +03:00\n" }
+
+  let(:agent) do
+    described_class.new user_agent: 'Scruber',
+                        proxy_id: 1,
+                        headers: {'a' => 1},
+                        cookie_jar: cookie_jar_string,
+                        disable_proxy: true
+  end
+
+  describe "initialize" do
+    it "should generate id" do
+      expect(agent.id).not_to be_nil
+    end
+  end
+
+  describe "save" do
+    it "should be stored to memory collection" do
+      agent.save
+      expect(Scruber::Helpers::FetcherAgentAdapters::Memory._collection[agent.id]).to eq(agent)
+    end
+  end
+
+  describe "delete" do
+    it "should be deleted from memory collection" do
+      agent.save
+      expect(Scruber::Helpers::FetcherAgentAdapters::Memory._collection[agent.id]).to eq(agent)
+      agent.delete
+      expect(Scruber::Helpers::FetcherAgentAdapters::Memory._collection[agent.id]).to be_nil
+    end
+  end
+
+  describe "class methods" do
+    describe "find" do
+      it "should find agent by id" do
+        agent.save
+        expect(Scruber::Helpers::FetcherAgentAdapters::Memory.find(agent.id)).to eq(agent)
+      end
+    end
+  end
+end

data/spec/helpers/proxy_rotator/proxy_rotator_proxy_spec.rb

@@ -0,0 +1,21 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::ProxyRotator::Proxy do
+
+  describe "proxy" do
+    let(:proxy) { described_class.new('127.0.0.1:3000', :user=>'user', :password=>'password', :probability=>1, :type=>'socks') }
+
+    it 'should have valid id' do
+      proxy = described_class.new('127.0.0.1:3000')
+      expect(proxy.id).to eq('127.0.0.1:3000')
+      proxy = described_class.new('127.0.0.1', port: 3001)
+      expect(proxy.id).to eq('127.0.0.1:3001')
+    end
+
+    it 'should raise error if port or address not given' do
+      expect{ described_class.new('127.0.0.1') }.to raise_error(Scruber::ArgumentError)
+      expect{ described_class.new('', port: 3000) }.to raise_error(Scruber::ArgumentError)
+    end
+  end
+
+end

data/spec/helpers/proxy_rotator/proxy_rotator_spec.rb

@@ -0,0 +1,118 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::ProxyRotator do
+
+  describe "configurable" do
+    before do
+      described_class.configure do
+        set_mode :random
+
+        add "127.0.0.1:3000"
+        add "127.0.0.1", port: 3001
+      end
+    end
+    
+    it "adds 2 proxies to list" do
+      expect(described_class.configuration.proxies.count).to eq(2)
+    end
+
+    it "clean proxies list" do
+      described_class.configure do
+        clean
+      end
+      expect(described_class.configuration.proxies.count).to eq(0)
+    end
+
+    it "should have random mode by default" do
+      expect(described_class.configuration.mode).to eq(:random)
+    end
+
+    it "should raise error when set incorrect mode" do
+      expect{ described_class.configure { set_mode :bad } }.to raise_error(Scruber::ArgumentError)
+    end
+
+    it "should build proxy_keys" do
+      described_class.configure do
+        clean
+        set_mode :round_robin
+
+        add "127.0.0.1:3000"
+        add "127.0.0.1", port: 3001
+      end
+      expect(described_class.configuration.proxy_keys.sort).to eq(["127.0.0.1:3000", "127.0.0.1:3001"].sort)
+    end
+
+    it "should rebuild proxy_keys" do
+      described_class.configure do
+        clean
+        set_mode :round_robin
+
+        add "127.0.0.1:3000"
+        add "127.0.0.1", port: 3001
+      end
+      expect(described_class.configuration.proxy_keys.sort).to eq(["127.0.0.1:3000", "127.0.0.1:3001"].sort)
+      described_class.configure do
+        add "127.0.0.5:3000"
+      end
+      expect(described_class.configuration.proxy_keys.sort).to eq(["127.0.0.1:3000", "127.0.0.1:3001", "127.0.0.5:3000"].sort)
+    end
+  end
+
+  describe "round_robin mode" do
+    before do
+      described_class.configure do
+        clean
+        set_mode :round_robin
+
+        add "127.0.0.1:3000"
+        add "127.0.0.2:3000"
+        add "127.0.0.3:3000"
+      end
+    end
+
+    it "should return all 3 proxies" do
+      expect(3.times.map{ described_class.next.host }.sort).to eq(["127.0.0.1", "127.0.0.2", "127.0.0.3"].sort)
+    end
+
+    it "should return all 3 proxies for random method" do
+      expect(3.times.map{ described_class.random.host }.sort).to eq(["127.0.0.1", "127.0.0.2", "127.0.0.3"].sort)
+    end
+
+    it "should return all 3 proxies twice" do
+      expect(6.times.map{ described_class.next.host }.sort).to eq(["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.1", "127.0.0.2", "127.0.0.3"].sort)
+    end
+  end
+
+  describe "random mode" do
+    before do
+      described_class.configure do
+        clean
+        set_mode :random
+
+        add "127.0.0.1:3000"
+        add "127.0.0.2:3000"
+        add "127.0.0.3:3000"
+      end
+    end
+
+    it "should return all 3 proxies (may raise phantom error)" do
+      expect(100.times.map{ described_class.next.host }.uniq.sort).to eq(["127.0.0.1", "127.0.0.2", "127.0.0.3"].sort)
+    end
+
+    it "should return 127.0.0.1 more often (may raise phantom error)" do
+      described_class.configure do
+        clean
+        set_mode :random
+
+        add "127.0.0.1:3000", probability: 0.9
+        add "127.0.0.2:3000", probability: 0.05
+        add "127.0.0.3:3000", probability: 0.05
+      end
+      expect(100.times.map{ described_class.next.host }.select{|h| h == '127.0.0.1'}.count).to be > 75
+    end
+  end
+
+  it "should be accessible from scraper" do
+    expect { Scruber.run(:sample) { ProxyRotator } }.not_to raise_error
+  end
+end

data/spec/helpers/user_agent_rotator/user_agent_rotator_spec.rb

@@ -0,0 +1,145 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::UserAgentRotator do
+
+  describe "configurable" do
+    context "with block" do
+      before do
+        described_class.configure do
+          clean
+          set_filter :all
+
+          add "Scruber 1.0", tags: [:robot, :scruber]
+          add "GoogleBot 1.0", tags: [:robot, :google]
+          add "Chrome 1.0", tags: [:desktop, :chrome]
+          add "Android 1.0", tags: [:mobile, :android]
+        end
+      end
+      
+      it "adds 4 user agents to list" do
+        expect(described_class.configuration.user_agents.count).to eq(4)
+      end
+
+      it "clean user agents list" do
+        described_class.configure do
+          clean
+        end
+        expect(described_class.configuration.user_agents.count).to eq(0)
+      end
+
+      it "should have all filter by default" do
+        expect(described_class.configuration.tags).to eq(:all)
+      end
+
+      it "should set different filter" do
+        described_class.configure do
+          set_filter :desktop
+        end
+        expect(described_class.configuration.tags).to eq(:desktop)
+      end
+    end
+
+    context "with dictionary" do
+      before do
+        Scruber::Core::Extensions::Loop.add_dictionary(:default_user_agents, File.expand_path(File.dirname(__FILE__))+'/user_agents.xml', :xml)
+        
+        described_class.configure do
+          clean
+          set_filter :all
+
+          loop :default_user_agents do |ua|
+            add ua['name'], tags: ua['tags'].split(',').map(&:strip)
+          end
+        end
+      end
+      
+      it "adds 4 user agents to list" do
+        expect(described_class.configuration.user_agents.count).to eq(4)
+      end
+
+      it "clean proxies list" do
+        described_class.configure do
+          clean
+        end
+        expect(described_class.configuration.user_agents.count).to eq(0)
+      end
+
+      it "should have all filter by default" do
+        expect(described_class.configuration.tags).to eq(:all)
+      end
+
+      it "should set different filter" do
+        described_class.configure do
+          set_filter :desktop
+        end
+        expect(described_class.configuration.tags).to eq(:desktop)
+      end
+    end
+  end
+
+  describe "with default config" do
+    before do
+      described_class.configure do
+        clean
+        set_filter :all
+
+        add "Scruber 1.0", tags: [:robot, :scruber]
+        add "GoogleBot 1.0", tags: [:robot, :google]
+        add "Chrome 1.0", tags: [:desktop, :chrome]
+        add "Android 1.0", tags: [:mobile, :android]
+      end
+    end
+
+    it "should return all 4 user agents" do
+      expect(4.times.map{ described_class.next }.sort).to eq(["Scruber 1.0","GoogleBot 1.0","Chrome 1.0","Android 1.0"].sort)
+    end
+
+    it "should return all 4 user agents twice" do
+      expect(8.times.map{ described_class.next }.sort).to eq(["Scruber 1.0","GoogleBot 1.0","Chrome 1.0","Android 1.0","Scruber 1.0","GoogleBot 1.0","Chrome 1.0","Android 1.0"].sort)
+    end
+
+    it "should return only robot user agents" do
+      described_class.configure do
+        set_filter :robot
+      end
+      expect(4.times.map{ described_class.next }.sort).to eq(["Scruber 1.0","GoogleBot 1.0","Scruber 1.0","GoogleBot 1.0"].sort)
+    end
+
+    it "should return only desktop chrome" do
+      described_class.configure do
+        set_filter [:desktop, :chrome]
+      end
+      expect(2.times.map{ described_class.next }.sort).to eq(["Chrome 1.0", "Chrome 1.0"].sort)
+    end
+  end
+
+  describe "with passed config" do
+    before do
+      described_class.configure do
+        clean
+        set_filter :bad
+
+        add "Scruber 1.0", tags: [:robot, :scruber]
+        add "GoogleBot 1.0", tags: [:robot, :google]
+        add "Chrome 1.0", tags: [:desktop, :chrome]
+        add "Android 1.0", tags: [:mobile, :android]
+      end
+    end
+
+    it "should return all 4 user agents" do
+      expect(4.times.map{ described_class.next(:all) }.sort).to eq(["Scruber 1.0","GoogleBot 1.0","Chrome 1.0","Android 1.0"].sort)
+    end
+
+    it "should return all 4 user agents twice" do
+      expect(8.times.map{ described_class.next(:all) }.sort).to eq(["Scruber 1.0","GoogleBot 1.0","Chrome 1.0","Android 1.0","Scruber 1.0","GoogleBot 1.0","Chrome 1.0","Android 1.0"].sort)
+    end
+
+    it "should return only robot user agents" do
+      expect(4.times.map{ described_class.next(:robot) }.sort).to eq(["Scruber 1.0","GoogleBot 1.0","Scruber 1.0","GoogleBot 1.0"].sort)
+    end
+
+    it "should return only desktop chrome" do
+      expect(2.times.map{ described_class.next([:desktop, :chrome]) }.sort).to eq(["Chrome 1.0", "Chrome 1.0"].sort)
+    end
+  end
+end

data/spec/helpers/user_agent_rotator/user_agent_rotator_user_agent_spec.rb

@@ -0,0 +1,40 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::UserAgentRotator::UserAgent do
+
+  describe "user_agent" do
+    let(:user_agent) { described_class.new('Scruber 1.0', tags: [:robot, :scruber]) }
+
+    it 'should have valid id' do
+      expect(user_agent.id).to eq('Scruber 1.0')
+    end
+
+    it 'should have valid tags' do
+      expect(user_agent.tags).to eq([:robot, :scruber])
+    end
+
+    it 'should always have array of tags' do
+      ua = described_class.new('Scruber 1.0')
+      expect(ua.tags).to eq([])
+
+      ua = described_class.new('Scruber 1.0', tags: nil)
+      expect(ua.tags).to eq([])
+
+      ua = described_class.new('Scruber 1.0', tags: :robot)
+      expect(ua.tags).to eq([:robot])
+    end
+
+    it 'should always have array of symbolic tags' do
+      ua = described_class.new('Scruber 1.0', tags: 'robot')
+      expect(ua.tags).to eq([:robot])
+
+      ua = described_class.new('Scruber 1.0', tags: ['robot', 'scruber'])
+      expect(ua.tags).to eq([:robot, :scruber])
+    end
+
+    it 'should raise error if port or address not given' do
+      expect{ described_class.new('') }.to raise_error(Scruber::ArgumentError)
+    end
+  end
+
+end

data/spec/helpers/user_agent_rotator/user_agents.xml

@@ -0,0 +1,6 @@
+<items>
+  <item name="Scruber 1.0" tags="robot,scruber" />
+  <item name="GoogleBot 1.0" tags="robot,google" />
+  <item name="Chrome 1.0" tags="chrome,desktop,macos" />
+  <item name="Android 1.0" tags="mobile,android" />
+</items>

data/spec/queue_adapter/memory_spec.rb

@@ -0,0 +1,15 @@
+require "spec_helper"
+
+RSpec.describe Scruber::QueueAdapters::Memory do
+  let(:queue){ described_class.new }
+  
+  it_behaves_like "queue_adapter"
+
+  it "shift first enqueued page" do
+    queue.add "http://example.com"
+    queue.add "http://example2.com"
+    page = queue.fetch_pending
+    expect(page.url).to eq("http://example.com")
+  end
+
+end

data/spec/queue_spec.rb

@@ -0,0 +1,27 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Queue do
+
+  describe "add_adapter" do
+    it "should raise error" do
+      expect{ described_class.add_adapter(:obj, Object) }.to raise_error(NoMethodError)
+    end
+
+    it "should add new adapter and return added class" do
+      expect(described_class.add_adapter(:simple2, Scruber::QueueAdapters::Memory)).to eq(Scruber::QueueAdapters::Memory)
+      expect(described_class._adapters.keys).to include(:simple2)
+    end
+  end
+
+  describe "adapter" do
+    it "should return default adapter" do
+      expect(described_class.adapter).to eq(Scruber::QueueAdapters::Memory)
+    end
+  end
+
+  describe "new" do
+    it "should return instance of default adapter" do
+      expect(described_class.new).to be_a(Scruber::QueueAdapters::Memory)
+    end
+  end
+end