ro_crawler 0.0.4

data/spec/ro_crawler/base_spec.rb

@@ -0,0 +1,197 @@
+require 'spec_helper'
+
+describe RoCrawler::Base do
+  before do
+    @b = RoCrawler::Base.new
+  end
+
+  after do
+    @b.browser_close
+  end
+
+  it 'local?' do
+    expect(@b.local?).to be_true
+  end
+
+  describe 'crawler' do
+    it 'phantomjs' do
+      expect do
+        @b.crawler driver: :phantomjs
+      end.not_to raise_error
+    end
+
+    it 'chrome' do
+      expect do
+        @b.crawler
+      end.not_to raise_error
+    end
+  end
+
+  it 'spider yjs' do
+    url = 'http://s.yingjiesheng.com/result.jsp?keyword=%E5%89%8D%E7%AB%AF%E5%AE%9E%E4%B9%A0&city=0&jobtype=0&do=1&stype=0'
+    expect do
+      ::RoCrawler::Base.new.spider(url, 'h3.title>a', '.job, .j_i')
+    end.not_to raise_error
+  end
+
+  describe 'spider v2ex' do
+    before do
+      @v2ex_spider = Base.new
+    end
+
+    #it 'case1' do
+    #  url = 'http://www.v2ex.com/go/jobs?p=1'
+    #  expect do
+    #    ::RoCrawler::Base.new.spider(url, 'span.item_title>a', '.topic_content')
+    #  end.not_to raise_error
+    #end
+
+    it 'case2' do
+      @v2ex_spider.instance_variable_set(:@link_titles, [
+          ["/t/79990#reply63", "[北京/杭州] 阿里巴巴2014校园招聘优秀人才内部同事推荐计划"]
+      ])
+      @v2ex_spider.instance_variable_set(:@home_url, "http://www.v2ex.com")
+      @v2ex_spider.instance_variable_set(:@intr_selector, ".cell>.topic_content")
+      @v2ex_spider.get_contents
+    end
+  end
+
+  it 'spider ruby-china' do
+    url = 'http://ruby-china.org/topics/node25'
+    expect do
+      ::RoCrawler::Base.new.spider(url, '.title>a', '.entry_content')
+    end.not_to raise_error
+  end
+
+  it 'spider ruby-china intr not nil' do
+    expect(
+        @b.get_tags_attrs_from('http://ruby-china.org/topics/13700', '.body.entry-content', 'inner_html')
+    ).not_to be_empty
+  end
+
+  it 'handler' do
+    ::RoCrawler::Base.class_eval do
+      def handler(&blk)
+        results = [
+            [1, 2, 3],
+            [4, 5, 6],
+            [7, 8, 9]
+        ]
+
+        handle results, &blk
+      end
+    end
+
+    ::RoCrawler::Base.new.handler do |a, b, c|
+      expect(a.to_s).to match /^\d$/
+      expect(b.to_s).to match /^\d$/
+      expect(c.to_s).to match /^\d$/
+    end
+  end
+
+
+  it 'get tags from url' do
+    expect(
+        @b.get_tags_from("http://baidu.com", 'body').inner_html
+    ).not_to be_nil
+  end
+
+  describe 'get attrs in tags' do
+    it 'when get two attribute' do
+      tags = @b.get_tags_from("http://baidu.com", '#m p#nv a')
+      @b.get_attrs_in(tags, 'href', 'text').each do |attrs|
+        expect(attrs[0][/http/]).to be_true
+        expect(attrs[1][/.*/]).to be_true
+      end
+    end
+
+    it 'when get a attribute' do
+      tag = @b.get_tags_from("http://baidu.com", 'a')
+      @b.get_attrs_in(tag, 'text').each do |attr|
+        expect(attr).to be_a String
+      end
+    end
+  end
+
+  describe 'get_tags_attrs_from' do
+    it 'case1' do
+      @b.get_tags_attrs_from("http://baidu.com", '#m p#nv a', 'href', 'text').each do |attrs|
+        expect(attrs[0][/http/]).to be_true
+        expect(attrs[1][/.*/]).to be_true
+      end
+    end
+
+    it 'case2' do
+      @b.get_tags_attrs_from("http://www.v2ex.com/t/80954#reply6", '.topic_content', 'text')
+    end
+  end
+
+  it 'handle result' do
+    results = [
+        [1, 2, 3],
+        [4, 5, 6],
+        [7, 8, 9]
+    ]
+
+    @b.handle results do |a, b, c|
+      expect(a.to_s).to match /^\d$/
+      expect(b.to_s).to match /^\d$/
+      expect(c.to_s).to match /^\d$/
+    end
+  end
+
+  it 'get home url' do
+    url = 'http://www.v2ex.com/go/jobs?p=1'
+    expect(@b.get_home_url(url)).to be == 'http://www.v2ex.com'
+    url = 'http://ruby-china.org/topics/node25'
+    expect(@b.get_home_url(url)).to be == 'http://ruby-china.org'
+  end
+
+  it 'handle accident error' do
+    # test refresh if timeout
+    ::RoCrawler::Base.class_eval do
+      def new_browser
+        @b = crawler.goto 'http://baidu.com'
+      end
+    end
+
+    b = @b.new_browser
+
+    expect do
+      @b.handle_accident_error do |browser|
+        10.times { browser.refresh }
+      end
+    end.not_to raise_error
+
+    b.close
+  end
+
+  describe 'get html from url' do
+    it 'refresh when timeout' do
+      ::RoCrawler::Base.class_eval do
+        def get_html_from(url)
+          html = ""
+          @b ||= crawler
+
+          get_html = lambda do
+            if url[/http/]
+              Nokogiri::HTML.parse @b.goto(url).html
+            else
+              Nokogiri::HTML.parse File.read(url)
+            end
+          end
+
+          begin
+            raise Net::ReadTimeout.new
+          rescue => e
+            html = get_html.call if e.is_a?(Net::ReadTimeout)
+          end
+
+          html
+        end
+      end
+
+      expect(@b.get_html_from('http://baidu.com')).not_to be_nil
+    end
+  end
+end

data/spec/ro_crawler/misc_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+include RoCrawler::Misc
+describe RoCrawler::Misc do
+  it 'urlify' do
+    baidu_url = Regexp.new "^http://baidu.com$"
+    local_url = Regexp.new "^file:///home/zxr/testfile$"
+    expect('baidu.com'.urlify).to match baidu_url
+    expect('/home/zxr/testfile'.urlify local:true).to match local_url
+  end
+end

data/spec/ro_crawler/watir/browser_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+include RoCrawler::Watir
+describe ::Watir::Browser do
+  it 'new' do
+    b = ::Watir::Browser.new :chrome
+    expect(b.goto 'http://google.com').to be_a(::Watir::Browser)
+  end
+end

data/spec/ro_crawler/watir/element_collection_spec.rb

@@ -0,0 +1,9 @@
+#require 'spec_helper'
+#
+#include WatirDSL
+#
+#describe ::Watir::ElementCollection do
+#  it 'collect' do
+#
+#  end
+#end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift File.expand_path('../..', __FILE__)
+require 'ro_support'
+require 'lib/ro_crawler'
+require 'headless'
+include RoCrawler
+RSpec.configure do |config|
+  config.before(:all) do
+    Headless.new.start
+  end
+
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: ro_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- ro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: watir-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: headless
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: grit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- rolobogu@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/ro_crawler.rb
+- lib/ro_crawler/watir.rb
+- lib/ro_crawler/version.rb
+- lib/ro_crawler/misc.rb
+- lib/ro_crawler/base.rb
+- lib/ro_crawler/watir/element.rb
+- lib/ro_crawler/watir/element_collection.rb
+- lib/ro_crawler/watir/browser.rb
+- lib/ro_crawler/methods.rb
+- MIT-LICENSE
+- Rakefile
+- README.rdoc
+- spec/fixtures/intr.html
+- spec/fixtures/list.html
+- spec/spec_helper.rb
+- spec/ro_crawler/watir/browser_spec.rb
+- spec/ro_crawler/watir/element_collection_spec.rb
+- spec/ro_crawler/misc_spec.rb
+- spec/ro_crawler/base_spec.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: ''
+test_files:
+- spec/fixtures/intr.html
+- spec/fixtures/list.html
+- spec/spec_helper.rb
+- spec/ro_crawler/watir/browser_spec.rb
+- spec/ro_crawler/watir/element_collection_spec.rb
+- spec/ro_crawler/misc_spec.rb
+- spec/ro_crawler/base_spec.rb