scruber 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4661c429c6b33a12841569c62835cc587e7f9464
-  data.tar.gz: 9cb87a48248746b30d2ece9db71136560629bdcb
+  metadata.gz: b845332207b108efa91983b4721cf7631120ea36
+  data.tar.gz: 4c9d931ccdbf777c9d469d7cf2697f00acfe94dc
 SHA512:
-  metadata.gz: c86674726ae45109383e8d0712612895136384eba1414e061c7d1de3dd9b699af6ab4022de037a537b1377a95b46be2aba10b4e9cea1664449ec880ae1b8189d
-  data.tar.gz: 738d34f37fdc629eb3f4755fffeb70e640e53bd7e33333357af0f9af286d648dd92846220a7ef0f9d06918e412fdf8c801a0b25e15cf74b912b0e0459e04009d
+  metadata.gz: b7c9bc638e7f168401bfd15de02746b691c5477563404db66810f894e7a1925821064b096d47c065209223856782e787bdc3f6272a179068aa0fdfcb6c14994d
+  data.tar.gz: f4c73d7e94e795b32c647285f320cf9e25b9f6efa0c528aaa3138ab99c263627f9ad47088851126d2df07421e633037fa3e8f434d7303a91ad350804d9a64903

data/lib/scruber/version.rb

@@ -1,3 +1,3 @@
 module Scruber
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

data/scruber.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   end
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
+    f.match(%r{^(test|features)/})
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }

data/spec/core/extensions/csv_output_spec.rb

@@ -0,0 +1,44 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Core::Extensions::CsvOutput do
+
+  describe "register" do
+    it "should extend Scruber::CsvOutput with csv_file and csv_out method" do
+      described_class.register
+
+      expect(Scruber::Core::Crawler.method_defined?(:csv_file)).to be_truthy
+      expect(Scruber::Core::Crawler.method_defined?(:csv_out)).to be_truthy
+      expect(Scruber::Core::Crawler._registered_method_missings.keys.include?(/\Acsv_(\w+)_file\Z/)).to be_truthy
+      expect(Scruber::Core::Crawler.new(:sample).respond_to?(:csv_products_file)).to be_truthy
+    end
+  end
+
+  describe "csv_file" do
+    it "should create csv_file and write output" do
+      described_class.register
+      csv_file_name = File.join(File.expand_path(File.dirname(__FILE__)), 'test.csv')
+
+      Scruber.run :sample do
+        csv_file csv_file_name, col_sep: '|'
+        csv_out [1,2,3]
+      end
+      expect(File.exists?(csv_file_name)).to be_truthy
+      expect(File.open(csv_file_name, 'r'){|f| f.read }.strip).to eq('1|2|3')
+      File.delete(csv_file_name) if File.exists?(csv_file_name)
+    end
+  end
+
+  describe "csv_{pattern}_file" do
+    it "should register file and write output" do
+      described_class.register
+      csv_file_name = File.join(File.expand_path(File.dirname(__FILE__)), 'products.csv')
+      Scruber.run :sample do
+        csv_products_file csv_file_name, col_sep: '|'
+        csv_products_out [1,2,3]
+      end
+      expect(File.exists?(csv_file_name)).to be_truthy
+      expect(File.open(csv_file_name, 'r'){|f| f.read }.strip).to eq('1|2|3')
+      File.delete(csv_file_name) if File.exists?(csv_file_name)
+    end
+  end
+end

data/spec/core/extensions/dict.csv

@@ -0,0 +1,4 @@
+zip,r10,country,state
+10001,true,US,NY
+54914,false,US,WI
+10002,true,US,NY

data/spec/core/extensions/log_spec.rb

@@ -0,0 +1,25 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Core::Extensions::Log do
+
+  describe "register" do
+    it "should extend Scruber::Core with log method" do
+      described_class.register
+
+      expect(Scruber::Core::Crawler.method_defined?(:log)).to be_truthy
+    end
+  end
+
+  describe "#log" do
+    let(:log_file) { Pathname.new(File.expand_path('../log.txt', __FILE__)) }
+    before { Scruber.logger = Logger.new(log_file)  }
+    after{ (File.delete(log_file) rescue nil) }
+
+    it "should write log to file" do
+      Scruber.run :sample, silent: true do
+        log "Seeding"
+      end
+      expect(File.open(log_file){|f| f.read}).to match(/Seeding/)
+    end
+  end
+end

data/spec/core/extensions/loop_spec.rb

@@ -0,0 +1,26 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Core::Extensions::Loop do
+
+  describe "register" do
+    it "should extend Scruber::Core with loop method" do
+      described_class.register
+
+      expect(Scruber::Core::Crawler.method_defined?(:loop)).to be_truthy
+    end
+
+    it "should add dictionary and read info" do
+      Scruber::Core::Extensions::Loop.register
+      $zip_codes = []
+      Scruber.run :sample do
+        add_dictionary :zip_codes_usa, File.expand_path(File.dirname(__FILE__))+'/dict.csv', :csv
+        seed do
+          loop :zip_codes_usa, state: 'NY' do |row|
+            $zip_codes.push row['zip']
+          end
+        end
+      end
+      expect($zip_codes).to eq(['10001', '10002'])
+    end
+  end
+end

data/spec/core/extensions/parser_aliases_spec.rb

@@ -0,0 +1,89 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Core::Extensions::ParserAliases do
+
+  describe "register" do
+    it "should extend Crawler with parse and parse_* methods" do
+      described_class.register
+
+      expect(Scruber::Core::Crawler.method_defined?(:parse)).to be_truthy
+      expect(Scruber::Core::Crawler._registered_method_missings.keys.include?(/\Aparse_(\w+)\Z/)).to be_truthy
+      expect(Scruber::Core::Crawler.new(:sample).respond_to?(:parse_product)).to be_truthy
+    end
+  end
+
+  describe "#parse" do
+    context "without format" do
+      it "should register parser" do
+        described_class.register
+
+        stub_request(:get, "http://example.com").to_return(body: 'Example Domain')
+
+        Scruber.run :sample do
+          get "http://example.com"
+          
+          parse do |page|
+            $page = page
+          end
+        end
+        expect($page.url).to eq("http://example.com")
+        expect($page.page_type.to_s).to eq("seed")
+      end
+
+      it "should register parser with custom page_type" do
+        described_class.register
+
+        stub_request(:post, "http://example.com").to_return(body: 'Example Domain')
+
+        Scruber.run :sample do
+          post_product "http://example.com"
+          
+          parse_product do |page|
+            $page = page
+          end
+        end
+        expect($page.url).to eq("http://example.com")
+        expect($page.method.to_s).to eq("post")
+        expect($page.page_type.to_s).to eq("product")
+      end
+    end
+
+    context "with format" do
+      it "should register parser" do
+        described_class.register
+
+        stub_request(:get, "http://example.com").to_return(body: '<div><span>Example Domain</span></div>')
+
+        Scruber.run :sample do
+          get "http://example.com"
+          
+          parse :html do |page,doc|
+            $page = page
+            $doc = doc
+          end
+        end
+        expect($doc.at('span').text).to eq("Example Domain")
+        expect($page.page_type.to_s).to eq("seed")
+        expect($page.method.to_s).to eq("get")
+      end
+
+      it "should register parser with custom page_type" do
+        described_class.register
+
+        stub_request(:post, "http://example.com").to_return(body: '<div><span>Example Post</span></div>')
+
+        Scruber.run :sample do
+          post_product "http://example.com"
+          
+          parse_product :html do |page,doc|
+            $page = page
+            $doc = doc
+          end
+        end
+        expect($doc.at('span').text).to eq("Example Post")
+        expect($page.method.to_s).to eq("post")
+        expect($page.page_type.to_s).to eq("product")
+      end
+    end
+  end
+end

data/spec/core/extensions/queue_aliases_spec.rb

@@ -0,0 +1,72 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Core::Extensions::QueueAliases do
+
+  describe "register" do
+    it "should extend Crawler with get,post,head and (get|post|head)_* methods" do
+      described_class.register
+
+      expect(Scruber::Core::Crawler.method_defined?(:get)).to be_truthy
+      expect(Scruber::Core::Crawler.method_defined?(:head)).to be_truthy
+      expect(Scruber::Core::Crawler.method_defined?(:post)).to be_truthy
+      expect(Scruber::Core::Crawler._registered_method_missings.keys.include?(/\A(get|post|head)_(\w+)\Z/)).to be_truthy
+      expect(Scruber::Core::Crawler.new(:sample).respond_to?(:get_product)).to be_truthy
+    end
+  end
+
+  describe "#get,#post" do
+    context "without options" do
+      it "should add page to queue" do
+        described_class.register
+
+        Scruber.run :sample do
+          get "http://example.com"
+          $page = queue.fetch_pending
+        end
+        expect($page.url).to eq("http://example.com")
+        expect($page.method.to_s).to eq("get")
+        expect($page.page_type.to_s).to eq("seed")
+      end
+
+      it "should add page to queue" do
+        described_class.register
+
+        Scruber.run :sample do
+          post_product "http://example.com"
+          $page = queue.fetch_pending
+        end
+        expect($page.url).to eq("http://example.com")
+        expect($page.method.to_s).to eq("post")
+        expect($page.page_type).to eq("product")
+      end
+    end
+
+    context "with options" do
+      it "should add page to queue" do
+        described_class.register
+
+        Scruber.run :sample do
+          get "http://example.com", user_agent: 'Agent 1'
+          $page = queue.fetch_pending
+        end
+        expect($page.url).to eq("http://example.com")
+        expect($page.method.to_s).to eq("get")
+        expect($page.page_type.to_s).to eq("seed")
+        expect($page.user_agent).to eq('Agent 1')
+      end
+
+      it "should add page to queue" do
+        described_class.register
+
+        Scruber.run :sample do
+          post_product "http://example.com", user_agent: 'Agent 1'
+          $page = queue.fetch_pending
+        end
+        expect($page.url).to eq("http://example.com")
+        expect($page.method.to_s).to eq("post")
+        expect($page.page_type).to eq("product")
+        expect($page.user_agent).to eq('Agent 1')
+      end
+    end
+  end
+end

data/spec/core/extensions/seed_spec.rb

@@ -0,0 +1,44 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Core::Extensions::Seed do
+
+  describe "register" do
+    it "should extend Scruber::Core with seed method" do
+      described_class.register
+
+      expect(Scruber::Core::Crawler.method_defined?(:seed)).to be_truthy
+    end
+  end
+
+  before do
+    stub_request(:get, "http://example.com").to_return(body: '<div><a>Main</a></div>')
+    stub_request(:get, "http://example.com/contacts").to_return(body: '<div><a>Contacts</a></div>')
+  end
+  
+  it "should execute seed block" do
+    $queue_size = 0
+    Scruber.run :sample do
+      seed do
+        get 'http://example.com'
+      end
+      $queue_size = queue.size
+    end
+    expect($queue_size).to eq(1)
+  end
+
+  it "should not execute seed block" do
+    $queue_size = 0
+    Scruber.run :sample do
+      seed do
+        get 'http://example.com'
+      end
+      seed do
+        get 'http://example.com/contacts'
+      end
+      $queue_size = queue.size
+      $page = queue.fetch_pending
+    end
+    expect($queue_size).to eq(1)
+    expect($page.url).to eq("http://example.com")
+  end
+end

data/spec/fetcher.rb

@@ -0,0 +1,27 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Fetcher do
+
+  describe "add_adapter" do
+    it "should raise error" do
+      expect{ described_class.add_adapter(:obj, Object) }.to raise_error(NoMethodError)
+    end
+
+    it "should add new adapter and return added class" do
+      expect(described_class.add_adapter(:typhoeus_fetcher, Scruber::FetcherAdapters::TyphoeusFetcher)).to eq(Scruber::FetcherAdapters::TyphoeusFetcher)
+      expect(described_class._adapters.keys).to include(:typhoeus_fetcher)
+    end
+  end
+
+  describe "adapter" do
+    it "should return default adapter" do
+      expect(described_class.adapter).to eq(Scruber::FetcherAdapters::TyphoeusFetcher)
+    end
+  end
+
+  describe "new" do
+    it "should return instance of default adapter" do
+      expect(described_class.new).to be_a(Scruber::FetcherAdapters::TyphoeusFetcher)
+    end
+  end
+end

data/spec/helpers/dictionary_reader/dict.csv

@@ -0,0 +1,4 @@
+postal_code,r10,country,state
+10001,true,US,NY
+54914,false,US,WI
+10002,true,US,NY

data/spec/helpers/dictionary_reader/dict.xml

@@ -0,0 +1,5 @@
+<items>
+  <item r10="true" country="US" state="NY" postal_code="10001" />
+  <item r10="false" country="US" state="WI" postal_code="54914" />
+  <item r10="true" country="US" state="NY" postal_code="10002" />
+</items>

data/spec/helpers/dictionary_reader/dict_records.xml

@@ -0,0 +1,5 @@
+<records>
+  <record r10="true" country="US" state="NY" postal_code="10001" />
+  <record r10="false" country="US" state="WI" postal_code="54914" />
+  <record r10="true" country="US" state="NY" postal_code="10002" />
+</records>

data/spec/helpers/dictionary_reader/dictionary_reader_csv_spec.rb

@@ -0,0 +1,36 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::DictionaryReader::Csv do
+
+  describe "register" do
+    it "should correctly read first element" do
+      cl = described_class.new(File.expand_path(File.dirname(__FILE__))+'/dict.csv')
+
+      result = nil
+      cl.read do |obj|
+        result = obj
+      end
+      expect(result.sort).to eq({"r10"=>"true", "country"=>"US", "state"=>"NY", "postal_code"=>"10002"}.sort)
+    end
+
+    it "should read 3 elements total" do
+      cl = described_class.new(File.expand_path(File.dirname(__FILE__))+'/dict.csv')
+
+      count = 0
+      cl.read do |obj|
+        count += 1
+      end
+      expect(count).to eq(3)
+    end
+
+    it "should read 1 elements with state=WI" do
+      cl = described_class.new(File.expand_path(File.dirname(__FILE__))+'/dict.csv')
+
+      results = []
+      cl.read({state: 'WI'}) do |obj|
+        results.push obj.sort
+      end
+      expect(results).to eq([{"r10"=>"false", "country"=>"US", "state"=>"WI", "postal_code"=>"54914"}.sort])
+    end
+  end
+end

data/spec/helpers/dictionary_reader/dictionary_reader_xml_spec.rb

@@ -0,0 +1,46 @@
+require "spec_helper"
+
+RSpec.describe Scruber::Helpers::DictionaryReader::Xml do
+
+  describe "register" do
+    it "should correctly read first element" do
+      cl = described_class.new(File.expand_path(File.dirname(__FILE__))+'/dict.xml')
+
+      result = nil
+      cl.read do |obj|
+        result = obj
+      end
+      expect(result.sort).to eq({"r10"=>"true", "country"=>"US", "state"=>"NY", "postal_code"=>"10002"}.sort)
+    end
+
+    it "should correctly read first element with different selector" do
+      cl = described_class.new(File.expand_path(File.dirname(__FILE__))+'/dict_records.xml')
+
+      result = nil
+      cl.read(selector: 'record') do |obj|
+        result = obj
+      end
+      expect(result.sort).to eq({"r10"=>"true", "country"=>"US", "state"=>"NY", "postal_code"=>"10002"}.sort)
+    end
+
+    it "should read 3 elements total" do
+      cl = described_class.new(File.expand_path(File.dirname(__FILE__))+'/dict.xml')
+
+      count = 0
+      cl.read do |obj|
+        count += 1
+      end
+      expect(count).to eq(3)
+    end
+
+    it "should read 1 elements with state=WI" do
+      cl = described_class.new(File.expand_path(File.dirname(__FILE__))+'/dict.xml')
+
+      results = []
+      cl.read({state: 'WI'}) do |obj|
+        results.push obj.sort
+      end
+      expect(results).to eq([{"r10"=>"false", "country"=>"US", "state"=>"WI", "postal_code"=>"54914"}.sort])
+    end
+  end
+end