ro_crawler 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 138a31aa60488f939724b71f2d74b2abcfa07e16
-  data.tar.gz: eb14725c6bce8e6150eaf11cfe66d13a120c934c
+  metadata.gz: 2bc31485c675436fdef8b64dfc058c8109509de6
+  data.tar.gz: 3090f40f7e0ce91a531b4b29c26fdda660e64988
 SHA512:
-  metadata.gz: 0b38276470529131ac0fd360f31a2cf232fc8d026d51a27abd815af4f203cef29733226b6e9d4c1e974e82611bea4869a63ce01d4b954f6cf57477818c0cbda4
-  data.tar.gz: 9fcdee015c33106ac2f82831136e6dddb314516143caab342a3e43bd460c33b95561cb00e9b2fbcca0987e6ed6fcf36a5dd74bc09cf6119eb188f1f90a0e9f68
+  metadata.gz: f8c5d1fa858d404688779e1b58618259736e27bc024b6a935c3fba53d32b9d70c720b875a524efe9b41a8da6bc0013ab13efa9624fb380adc84610bbd923f327
+  data.tar.gz: 2b71ebef44fec7ce565550c3322d9400087f0ed95733720153655a6931c553421d2429d4bf80fc67c8581a05c72bdf520ffed224bc4a7f76c79658bec12b40e0

data/lib/ro_crawler.rb

@@ -1,7 +1,5 @@
 # require all files in dir name is same with __FILE_-
-autoload :FileActions, 'ro_support/file_actions'
-autoload :Array, 'ro_support/array'
-autoload :Log, 'ro_support/log'
+require 'ro_support/array'
 
 include RoSupport::FileActions
 include RoSupport::Array

data/lib/ro_crawler/base.rb

@@ -18,8 +18,8 @@ module RoCrawler
 
     def get_contents
       ro_raise(err "@link_titles is nil", output: ['@url']) if @link_titles.nil?
-      @link_title_contents = @link_titles.dup
-      @link_title_contents.each do |link_content|
+      @offers = @link_titles.dup
+      @offers.each do |link_content|
         if link_content[0][/http/]
           link = link_content[0]
         else
@@ -28,14 +28,23 @@ module RoCrawler
 
         link_content << intr = get_tags_attrs_from(link, @intr_selector, 'text')
         unless intr.is_a? String
-          raise_log 'intr must be a string', 'ro_crawler_base.log'
+          puts_log 'intr must be a string', 'ro_crawler_base.log'
         end
         link_content
       end
     end
 
-    def handler(&blk)
-      handle @link_title_contents, &blk
+    def handler
+      ro_raise(err '@offers is empty') if @offers.empty?
+      if block_given?
+        if @offers.is_a?(Array)
+          @offers.each do |offer|
+            yield offer
+          end
+        else
+          raise "@offers is not a Array, @offers is #{@offers.class}"
+        end
+      end
     end
 
     def open_browser(driver)
@@ -126,11 +135,6 @@ module RoCrawler
     end
 
     def handle(results)
-      if results.is_a?(Array)
-        results.each do |result|
-          yield result
-        end
-      end
     end
 
     def get_home_url(url)

data/lib/ro_crawler/version.rb

@@ -1,3 +1,3 @@
 module RoCrawler
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/spec/ro_crawler/base_spec.rb

@@ -90,13 +90,13 @@ describe RoCrawler::Base do
   end
 
 
-  it 'get tags from url' do
+  it 'get_tags_from_url' do
     expect(
         @b.get_tags_from("http://baidu.com", 'body').inner_html
     ).not_to be_nil
   end
 
-  describe 'get attrs in tags' do
+  describe 'get_attrs_in_tags' do
     it 'when get two attribute' do
       tags = @b.get_tags_from("http://baidu.com", '#m p#nv a')
       @b.get_attrs_in(tags, 'href', 'text').each do |attrs|
@@ -121,9 +121,23 @@ describe RoCrawler::Base do
       end
     end
 
-    it 'case2' do
+    it 'v2ex' do
       @b.get_tags_attrs_from("http://www.v2ex.com/t/80954#reply6", '.topic_content', 'text')
     end
+
+    it 'ruby_china' do
+      url = "http://ruby-china.org/topics/node25"
+      anchor_selector = ".title>a"
+      results = @b.get_tags_attrs_from(url, anchor_selector, 'text')
+      expect(results.count).to be == 15
+    end
+
+    it 'yingjiesheng' do
+      url = "http://s.yingjiesheng.com/result.jsp?keyword=%E5%89%8D%E7%AB%AF&city=0&jobtype=0&do=1&stype=0"
+      anchor_selector = "h3.title>a"
+      results = @b.get_tags_attrs_from(url, anchor_selector, 'text')
+      expect(results.count).to be == 10
+    end
   end
 
   it 'handle result' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ro_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - ro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-26 00:00:00.000000000 Z
+date: 2013-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -24,6 +24,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: ro_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: term-ansicolor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: watir-rails
   requirement: !ruby/object:Gem::Requirement
@@ -125,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.1.5
 signing_key: 
 specification_version: 4
 summary: ''