roro_crawler 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c624947b38a0356ad347fe2305b5ca5b42b26a80
-  data.tar.gz: 17658d2929d346b2302448288876d48587ac88c9
+  metadata.gz: 9d5915884d8cf5013726ac53c6373dab6061f984
+  data.tar.gz: a83f6c9f6f7019b7d58a08588e63989b1fbf6ca9
 SHA512:
-  metadata.gz: 016a1722e2afbf2726515d71f7f1b4b26288ee3dc6b4035c7313ea49c129f9040c240754c0d69befa54793b1e7475b7ad61fac0d875c5db3be964338e6a7a4d6
-  data.tar.gz: eb792990a4f3f2935356771aef37cbadfdb066660fba4a5fbf0ee82ae3718b125bf7183cdaf37efa759cb56152c3bc59715659459146e806670610c81a5aba89
+  metadata.gz: b3b91e75d2b5cf527ec7c2bdd7d783ece4002fbf6374403dbe4eeeed551d12488ee7487b044602b585e71cf3f954a8a6702df5d5fd22fc4af08c8dbddae9b03e
+  data.tar.gz: a84a083837e112f18403a43e60b162b2430d2cb326614a2cee2578bb2d8849905dd386abc7348ff91797dfb052987f12b99a8d0a6d7c44c77c174dc196602279

data/Rakefile

@@ -16,7 +16,6 @@ end
 
 
 
-
 Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
@@ -30,3 +29,9 @@ end
 
 
 task default: :test
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new
+
+task default: :spec

data/lib/roro_crawler.rb

@@ -1,5 +1,7 @@
 # require all files in dir name is same with __FILE_-
-dirname = __FILE__.split('/').last.gsub(/\.rb/, '')
-Dir[File.expand_path("../#{dirname}/*", __FILE__)].each do |file|
-  require file
-end
+require 'roro_support'
+require 'headless'
+
+$LOAD_PATH.unshift File.expand_path('../roro_crawler', __FILE__)
+require 'methods'
+require 'base'

data/lib/roro_crawler/base.rb

@@ -0,0 +1,40 @@
+module RoRoCrawler
+  class Base
+
+    include RoRoSupport::Crawler
+
+    def spider(url, auchor_selector, intr_selector)
+      @url = url
+      @home_url = get_home_url(url)
+      @auchor_selector = auchor_selector
+      @intr_selector = intr_selector
+      get_link_titles
+      get_intrs
+      browser_close
+    end
+
+    def get_link_titles
+      @link_titles = get_tags_attrs_from(@url, @auchor_selector, 'href', 'text')
+    end
+
+    def get_intrs
+      raise "#{@link_titles} is nil" if @link_titles.nil?
+      @link_title_intrs = @link_titles.dup
+      @link_title_intrs.each do |link_content|
+        if link_content[0][/http/]
+          link = link_content[0]
+        else
+          link = "#{@home_url}#{link_content[0]}"
+        end
+
+        link_content << intr = get_tags_attrs_from(link, @intr_selector, 'text')
+        raise "crawler find multi intrs according #{@intr_selector}, please make @intr_selector more exactly, error intr is #{intr}" unless intr.is_a? String
+       link_content
+      end
+    end
+
+    def handler(&blk)
+      handle @link_title_intrs, &blk
+    end
+  end
+end

data/lib/roro_crawler/methods.rb

@@ -0,0 +1,18 @@
+#require 'nokogiri'
+#module RoRoCrawler
+#  module Methods
+#    include ::RoRoSupport::Crawler
+#    def get_intr_from(html)
+#      doc = Nokogiri::HTML.parse html
+#      doc.css('.jobIntro, .j_i')
+#      .gsub(/(本站提醒:如何识别虚假招聘信息？求职必看，切勿受骗上当！)|(如何写一份简单、直接、高效的求职信？)/, '')
+#    end
+#
+#    def get_links_contents_from(url, selector)
+#      as = @browser.css @link_selector
+#      unless as.nil?
+#        @offers = as.collect('text', 'href')
+#      end
+#    end
+#  end
+#end

data/lib/roro_crawler/version.rb

@@ -1,3 +1,3 @@
-module Crawler
-  VERSION = "0.0.3"
+module RoRoCrawler
+  VERSION = "0.0.4"
 end

data/spec/roro_crawler/base_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+describe ::RoRoCrawler::Base do
+  before do
+    #Headless.new.start
+  end
+  it 'spider yjs' do
+    url = 'http://s.yingjiesheng.com/result.jsp?keyword=%E5%89%8D%E7%AB%AF%E5%AE%9E%E4%B9%A0&city=0&jobtype=0&do=1&stype=0'
+    expect(
+        ::RoRoCrawler::Base.new.spider(url, 'h3.title>a', '.job, .j_i')
+    ).not_to raise_error
+  end
+
+  describe 'spider v2ex' do
+    before do
+      @v2ex_spider = Base.new
+    end
+
+    it 'case1' do
+      url = 'http://www.v2ex.com/go/jobs?p=1'
+      expect(
+          ::RoRoCrawler::Base.new.spider(url, 'span.item_title>a', '.topic_content')
+      ).not_to raise_error
+    end
+
+    it 'case2' do
+      @v2ex_spider.instance_variable_set(:@link_titles, [
+          ["/t/79990#reply63", "[北京/杭州] 阿里巴巴2014校园招聘优秀人才内部同事推荐计划"]
+      ])
+      @v2ex_spider.instance_variable_set(:@home_url, "http://www.v2ex.com")
+      @v2ex_spider.instance_variable_set(:@intr_selector, ".cell>.topic_content")
+      @v2ex_spider.get_intrs
+    end
+  end
+
+  it 'spider ruby-china' do
+    url = 'http://ruby-china.org/topics/node25'
+    expect(
+        ::RoRoCrawler::Base.new.spider(url, '.title>a', '.entry_content')
+    ).not_to raise_error
+  end
+
+  it 'spider ruby-china intr not nil' do
+    expect(
+        get_tags_attrs_from('http://ruby-china.org/topics/13700', '.body.entry-content', 'inner_html')
+    ).not_to be_empty
+  end
+
+  it 'handler' do
+    ::RoRoCrawler::Base.class_eval do
+      def handler(&blk)
+        results = [
+            [1, 2, 3],
+            [4, 5, 6],
+            [7, 8, 9]
+        ]
+
+        handle results, &blk
+      end
+    end
+
+    ::RoRoCrawler::Base.new.handler do |a, b, c|
+      expect(a.to_s).to match /^\d$/
+      expect(b.to_s).to match /^\d$/
+      expect(c.to_s).to match /^\d$/
+    end
+  end
+end

data/spec/roro_crawler_spec.rb

@@ -0,0 +1 @@
+require 'spec_helper'

data/spec/spec_helper.rb

@@ -1,13 +1,17 @@
-# This file was generated by the `rspec --init` command. Conventionally, all
-# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
-# Require this file using `require "spec_helper"` to ensure that it is only
-# loaded once.
-#
-# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 Bundler.require
-require 'active_support'
-require File.expand_path('../../lib/roro_crawler', __FILE__)
+require 'roro_support'
+require 'roro_crawler'
+include ::RoRoSupport::Crawler
+include ::RoRoCrawler
 RSpec.configure do |config|
-  config.before :all do
-  end
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: roro_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - roro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-25 00:00:00.000000000 Z
+date: 2013-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -24,26 +24,81 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: watir-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: headless
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: grit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email:
-- roro@gmail.com
+- zhuxingruo3@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/roro_crawler/crawler_handler.rb
 - lib/roro_crawler/version.rb
-- lib/roro_crawler/crawler_class.rb
+- lib/roro_crawler/base.rb
+- lib/roro_crawler/methods.rb
 - lib/roro_crawler.rb
-- lib/tasks/crawler_tasks.rake
 - MIT-LICENSE
 - Rakefile
 - README.rdoc
 - spec/fixtures/intr.html
 - spec/fixtures/list.html
 - spec/spec_helper.rb
-- spec/crawler_handler_spec.rb
-- spec/crawler_class_spec.rb
+- spec/roro_crawler/base_spec.rb
+- spec/roro_crawler_spec.rb
 homepage: ''
 licenses: []
 metadata: {}
@@ -71,5 +126,5 @@ test_files:
 - spec/fixtures/intr.html
 - spec/fixtures/list.html
 - spec/spec_helper.rb
-- spec/crawler_handler_spec.rb
-- spec/crawler_class_spec.rb
+- spec/roro_crawler/base_spec.rb
+- spec/roro_crawler_spec.rb

data/lib/roro_crawler/crawler_class.rb

@@ -1,98 +0,0 @@
-require 'roro_support'
-require 'active_support'
-require File.expand_path('../crawler_handler', __FILE__)
-
-include Crawler
-
-module Crawler
-  class Klass
-    attr_accessor :offers, :browser, :link, :title, :intr, :page
-
-    def initialize(options={})
-      @visible = options[:visible]
-      @link_selector = 'h3.title>a'
-      @offers = Hash.new
-      @page = 0
-
-      if @visible
-        @browser = crawler visible: @visible
-      else
-        @browser = crawler
-      end
-    end
-
-    def goto_next
-      @page += 1
-      @browser.goto "http://s.yingjiesheng.com/result.jsp?keyword=web&start=#{@page*10}&period=0&sort=score&jobtype=0"
-    end
-
-    def site(keyword, page_num, options={})
-      if url.nil?
-        p <<-MSG
-        please add
-def site
-  url = 'http://website.com'
-  super
-end
-        MSG
-      end
-      @browser.goto url
-
-      page_num.to_i.times do
-        links
-        link_contents
-        "http://s.yingjiesheng.com/result.jsp?keyword=web&start=#{@page*10}&period=0&sort=score&jobtype=0"
-        goto_next
-      end
-
-      @browser.close
-    end
-
-
-    def close
-      @browser.close
-    end
-
-
-    def links
-      as = @browser.css @link_selector
-      unless as.nil?
-        @offers = as.
-
-collect('text', 'href')
-      end
-    end
-
-    def link_contents
-      @offers.each do |title, link|
-        if title && link
-          @title = title
-          @link = link
-          @intr = msg link
-          safe_save
-        end
-      end
-    end
-
-    def safe_save
-      msg = <<-MSG
-      title: #{@title}
-      link: #{@link}
-      MSG
-
-      Rails.logger.info msg
-
-      p @intr
-      return if @intr.nil?
-      Offer.create(link: @link, title: @title, intr: @intr, from: 'yjs')
-    end
-
-
-    def msg(href)
-      if href[/http\:\/\/www\.yingjiesheng\.com\/job\-\w+/]
-        @browser.goto href
-        return Handler.get_intr_from(@browser.html)
-      end
-    end
-  end
-end

data/lib/roro_crawler/crawler_handler.rb

@@ -1,13 +0,0 @@
-require 'nokogiri'
-module Crawler
-  module Handler
-    class << self
-      def get_intr_from(html)
-        doc = Nokogiri::HTML.parse html
-        doc.css('.jobIntro, .j_i')
-        .text
-        .gsub(/(本站提醒:如何识别虚假招聘信息？求职必看，切勿受骗上当！)|(如何写一份简单、直接、高效的求职信？)/, '')
-      end
-    end
-  end
-end

data/lib/tasks/crawler_tasks.rake

@@ -1,4 +0,0 @@
-# desc "Explaining what the task does"
-# task :roro_crawler do
-#   # Task goes here
-# end

data/spec/crawler_class_spec.rb

@@ -1,23 +0,0 @@
-require 'spec_helper'
-require 'roro_support'
-
-describe 'Crawler' do
-  before do
-  end
-
-  after do
-    if @c.browser
-
-    end
-  end
-
-  describe "links" do
-    before do
-
-    end
-
-    it 'links can get links correctly' do
-
-    end
-  end
-end

data/spec/crawler_handler_spec.rb

@@ -1,21 +0,0 @@
-require 'spec_helper'
-include Crawler
-
-describe "Handler" do
-  describe 'get_intr_from' do
-    before :all do
-      @pass = lambda do
-        content = File.read(@fixtures[:intr])
-        intr = Handler::get_intr_from content
-        print intr
-        expect(intr.length).to be < 1000
-        expect(intr).not_to include "本站提醒:如何识别虚假招聘信息？求职必看，切勿受骗上当！"
-        expect(intr).not_to include "如何写一份简单、直接、高效的求职信？"
-      end
-
-    end
-    it 'pass spec1' do
-      @pass.call
-    end
-  end
-end