spider_rails 4.0.2 → 4.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e182b6084575d3b2ac7e4cc9dcaf9c07b7018fc
-  data.tar.gz: 060df4f018e1bf486c0ac03d923fde72f5cd3f96
+  metadata.gz: e9ae14d4aa7c198c8e42c9075b9cf76457c1d6a9
+  data.tar.gz: 1177b0a8dfe1c3715050a9bd5dcb25f14daaf32d
 SHA512:
-  metadata.gz: 386ffce6067639a68a264d4fe73d9d2eb398874d4c68f7290fbbf1bfd4163206e3c1e9754f7891b43b390dea38fefcd0674cbac92b7c3fb6bae55ea3fc94544e
-  data.tar.gz: 463612cd49d92066f7c349f4188a9cff34e3e31a69ebeb9ce1ce874e006e1aae8586b3d41a21a323138bcee6fa69a7c706d08105d5762c21a80a86bca1653436
+  metadata.gz: 3b1573e5ebed6b5bce30e04f715a0058d59d3eeaf8fad387a93fe8fecd71334a471cd946aee1d7f150f865ad8bc8c6ace4672b9baa2ceaaeff302eb5b2b44778
+  data.tar.gz: 29b0d0bcccd465fc61f11e704e655e505d024d2ac9646d415398cfea67c93a665ff33af37ba6b2f73133b9a1830346d1013a21e7d4ea99f3c862b9f13e61a321

data/Rakefile

@@ -19,3 +19,5 @@ Bundler::GemHelper.install_tasks
 require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/lib/spider_rails/{common.rb → common/common.rb}

@@ -1,28 +1,5 @@
-module Spider
+module Common
   class Common
-    def full_site
-
-    end
-
-    def full_site_filter
-
-    end
-
-    def full_page
-      close_all_chromes
-    end
-
-    def get element
-
-    end
-
-    def single(element)
-
-    end
-
-    def single_filter
-    end
-
     def get_content(element, selector, &block)
       begin
         if block_given?
@@ -37,11 +14,10 @@ module Spider
         end
       end
     end
-
   end
 
   class << self
-    def open_browser driver, url
+    def start driver, url
       #@browser = Watir::Browser.new :chrome, switches: %w( --user-data-dir=/home/zxr/.config/google-chrome)
       @browser = Watir::Browser.new driver
       @browser.goto url

data/lib/spider_rails/common/dsl.rb

@@ -0,0 +1,32 @@
+require 'watir-webdriver'
+require 'nokogiri'
+module DSL
+  # Add a visit method to ::Watir::Browser
+  class Browser < ::Watir::Browser
+    def visit(relative_url = nil, base_url = 'http://localhost:3000/')
+      goto("#{base_url}#{relative_url}")
+    end
+
+    def initialize(browser = :phantomjs, *args)
+      super
+    end
+
+    def dsl_enable
+      @doc = Nokogiri::HTML.parse(self.html)
+      ::String.class_variable_set(:@@doc, @doc)
+      eval <<-RUBY
+      class ::String
+        def ctn
+          if block_given?
+            @@doc.css(self) &block
+          else
+            @@doc.css(self).each do |e|
+              return e.text
+            end
+          end
+        end
+      end
+      RUBY
+    end
+  end
+end

data/lib/spider_rails/common/rspec.rb

@@ -0,0 +1,5 @@
+require File.expand_path('../dsl', __FILE__)
+
+RSpec.configure do |config|
+
+end

data/lib/spider_rails/{bilibili.rb → specific/bilibili.rb}

@@ -1,5 +1,5 @@
-module Spider
-  class BiliBili < Common
+module Specific
+  class BiliBili
 
     def get_res
       animations = Hash.new

data/lib/spider_rails/specific/google_dict.rb

@@ -0,0 +1,91 @@
+  module Specific
+    class GoogleDict
+      def get_cards(keywords)
+        Headless.new.start
+        keywords.each do |keyword|
+          unless Card.find_by_word(keyword)
+            get_card keyword
+            save_record(Card, word: @card[:Word],
+                        voice: @card[:Voice],
+                        verb: @card[:Verb],
+                        adj: @card[:Adjective],
+                        noun: @card[:Noun],
+                        pronoun: @card[:Pronoun],
+                        synonyms: @card[:Synonyms],
+                        abbr: @card[:Abbreviation],
+                        prep: @card[:Preposition],
+                        conj: @card[:Conjunction]
+            )
+          end
+        end
+      end
+
+      def get_keywords(path)
+        f = File.new(path)
+        dict = f.read.split(/\W/)
+        dict.delete("")
+        dict.uniq!
+        dict
+      end
+
+      class << self
+        def alias_methods(*args)
+          args.each do |arg|
+            alias_method arg, args.last
+          end
+        end
+      end
+
+      def get_card keyword
+        @card = Hash.new
+        @page = start "https://www.google.com.hk/search?newwindow=1&safe=strict&q=#{keyword}+define&oq=#{keyword}+define"
+
+        doc = Nokogiri::HTML.parse @page.html
+
+        GoogleDict.alias_methods :card, :voice, :word, :get_content
+        card(doc, 'li.dct') do |c|
+          @card[:Word] = keyword.downcase
+          @card[:Voice] = voice(c, 'h3+.vk_sh')
+
+          # Get word explainations
+          get_explain(c)
+        end
+
+        @page.close
+        @card.delete(0)
+        @card
+      end
+
+      def get_explain(c)
+        type_nodes = c.css('div.vk_gy.vk_sh').to_a
+        content_nodes = c.css('div.vk_gy.vk_sh+div').to_a
+        type_nodes.each_with_index do |t, i|
+          table = content_nodes[i]
+          if table.css('li').count >= 2
+            fin_content = Array.new
+            table.css('li').each do |l|
+              fin_content << l.content
+            end
+          else
+            fin_content = table.content
+          end
+          @card[t.text.to_sym] = fin_content
+        end
+      end
+
+      #def login(username, password)
+      #  element?('a.gbgt#gb_70') { |e| e.click }
+      #  @b.text_field(name: 'Email').set username
+      #  @b.text_field(name: 'Passwd').set password
+      #  element?('input#signIn') { |e| e.click }
+      #end
+
+      def element?(selector, &block)
+        e = @page.element(css: selector)
+        if yield e
+        else
+          'element is nil'
+        end
+      end
+    end
+  end

data/lib/spider_rails/{ji_ying.rb → specific/ji_ying.rb}

@@ -1,5 +1,10 @@
-module Spider
-  class JiYing < Common
+module Specific
+  # Download JiYing resources
+  # example:
+  #    @page = ::Spider.open_browser(:phantomjs, 'http://bt.ktxp.com/sort-50-1.html')
+  #    jy = ::Spider::JiYing.new(@page)
+  #    jy.full_site
+  class JiYing
     attr_accessor :ani, :anis, :page
 
     def initialize page
@@ -21,19 +26,23 @@ module Spider
         (1..fp).each do |page_num|
           full_page page_num
         end
+        p_anis
       rescue Exception
         raise %Q(page isn't not exist)
       end
     end
 
     def multi_pages final_page_num
-      begin
-        (1..final_page_num).each do |page_num|
-          full_page page_num
-        end
+      (1..final_page_num).each do |page_num|
+        full_page page_num
+        p_anis
       end
     end
 
+    def p_anis
+      p "@anis is #{@anis}"
+    end
+
     def final_page
       if @mode == 'search'
         fp = @page.element(css: '.title h2 a').text[/\(.+\)/].gsub!(/\(|\)/, '').to_i/100 + 1
@@ -51,7 +60,7 @@ module Spider
         when 'normal'
           @page.goto "#{@base_url}#{page_num}.html"
       end
-      html = Nokogiri::HTML.parse @page.html
+      html = ::Nokogiri::HTML.parse @page.html
 
       html.css('.ltext').each do |td|
         single(td)
@@ -70,11 +79,13 @@ module Spider
       @ani[:title],
           @ani[:size],
           @ani[:finish] = get_content(element, 'a.quick-down+a', 'td.ltext+td', 'td.ltext+td+td+td+td')
-      @anis << @ani
+      p "Get Animation: #{@ani[:title]}"
+
+      @anis << @ani.dup
     end
 
-    def count
-      self.anis.count
+    def ani_count
+      @anis.uniq.count if @anis
     end
 
     def get_content(element, *selectors)

data/lib/spider_rails/version.rb

@@ -1,3 +1,3 @@
 module SpiderRails
-  VERSION = "4.0.2"
+  VERSION = "4.0.3"
 end

data/lib/spider_rails.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 current_file_name = __FILE__.split('/').last.gsub('.rb', '')
-Dir[File.expand_path("../#{current_file_name}/*.rb", __FILE__)].each { |file| require file }
+Dir[File.expand_path("../#{current_file_name}/**/*.rb", __FILE__)].each { |file| require file }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spider_rails
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.0.3
 platform: ruby
 authors:
 - zhuxingruo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-20 00:00:00.000000000 Z
+date: 2013-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -38,6 +38,104 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: watir
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: headless
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: spork
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: nil
 email:
 - zhuxingruo3@gmail.com
@@ -45,14 +143,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/spider_rails/common.rb
 - lib/spider_rails/version.rb
-- lib/spider_rails/bilibili.rb
-- lib/spider_rails/sample_data.rb
-- lib/spider_rails/spread_sheet.rb
-- lib/spider_rails/google_dict.rb
-- lib/spider_rails/ji_ying.rb
-- lib/spider_rails/hl.rb
+- lib/spider_rails/specific/bilibili.rb
+- lib/spider_rails/specific/spread_sheet.rb
+- lib/spider_rails/specific/google_dict.rb
+- lib/spider_rails/specific/ji_ying.rb
+- lib/spider_rails/common/common.rb
+- lib/spider_rails/common/rspec.rb
+- lib/spider_rails/common/dsl.rb
 - lib/spider_rails.rb
 - MIT-LICENSE
 - Rakefile

data/lib/spider_rails/google_dict.rb

@@ -1,91 +0,0 @@
-module Spider
-  class GoogleDict < Common
-    def get_cards(keywords)
-      Headless.new.start
-      keywords.each do |keyword|
-        unless Card.find_by_word(keyword)
-          get_card keyword
-          save_record(Card, word: @card[:Word],
-                      voice: @card[:Voice],
-                      verb: @card[:Verb],
-                      adj: @card[:Adjective],
-                      noun: @card[:Noun],
-                      pronoun: @card[:Pronoun],
-                      synonyms: @card[:Synonyms],
-                      abbr: @card[:Abbreviation],
-                      prep: @card[:Preposition],
-                      conj: @card[:Conjunction]
-          )
-        end
-      end
-    end
-
-    def get_keywords(path)
-      f = File.new(path)
-      dict = f.read.split(/\W/)
-      dict.delete("")
-      dict.uniq!
-      dict
-    end
-
-    class << self
-      def alias_methods(*args)
-        args.each do |arg|
-          alias_method arg, args.last
-        end
-      end
-    end
-
-    def get_card keyword
-      @card = Hash.new
-      @page = open_browser "https://www.google.com.hk/search?newwindow=1&safe=strict&q=#{keyword}+define&oq=#{keyword}+define"
-
-      doc = Nokogiri::HTML.parse @page.html
-
-      GoogleDict.alias_methods :card, :voice, :word, :get_content
-      card(doc, 'li.dct') do |c|
-        @card[:Word] = keyword.downcase
-        @card[:Voice] = voice(c, 'h3+.vk_sh')
-
-        # Get word explainations
-        get_explain(c)
-      end
-
-      @page.close
-      @card.delete(0)
-      @card
-    end
-
-    def get_explain(c)
-      type_nodes = c.css('div.vk_gy.vk_sh').to_a
-      content_nodes = c.css('div.vk_gy.vk_sh+div').to_a
-      type_nodes.each_with_index do |t, i|
-        table = content_nodes[i]
-        if table.css('li').count >= 2
-          fin_content = Array.new
-          table.css('li').each do |l|
-            fin_content << l.content
-          end
-        else
-          fin_content = table.content
-        end
-        @card[t.text.to_sym] = fin_content
-      end
-    end
-
-    #def login(username, password)
-    #  element?('a.gbgt#gb_70') { |e| e.click }
-    #  @b.text_field(name: 'Email').set username
-    #  @b.text_field(name: 'Passwd').set password
-    #  element?('input#signIn') { |e| e.click }
-    #end
-
-    def element?(selector, &block)
-      e = @page.element(css: selector)
-      if yield e
-      else
-        'element is nil'
-      end
-    end
-  end
-end

data/lib/spider_rails/hl.rb

@@ -1,13 +0,0 @@
-module Lib
-  module Hl
-    class << self
-      def run
-        h = Headless.new
-        h.start
-        b = Watir::Browser.new :chrome, switches: %w[--proxy-server=socks5://127.0.0.1:7070]
-        b.goto 'https://www.google.com.hk/search?q=google+define&oq=google+define'
-        p b.title
-      end
-    end
-  end
-end

data/lib/spider_rails/sample_data.rb

@@ -1,10 +0,0 @@
-module Spider
-  class SampleData < Common
-    def generate
-      100.times do |n|
-        Novel.create(title: "やめて#{n}", content: 'やめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめて')
-        Card.create(word: "やめて#{n}", voice: 'やめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめて')
-      end
-    end
-  end
-end