spider_rails 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 42525540b17a7f35194db3126b79a8c0183e9090
+  data.tar.gz: a9d09421f94b4555b97dd098093e211d9c33c7b7
+SHA512:
+  metadata.gz: 62779738390335964ea5b8f596795b69820619e5cc60b800e604fe19292ac8099a42804e2207903f931afcb7d2b2a27439c8a8811aec4b39121334c820232633
+  data.tar.gz: d5e18b777d1a44726c4093b8b50cd83b9ef6d94fd606581ac302dfd110f5eec692c802bb15c01cda764cf80ce202230c5a426006647add04df31a18a7b8a9606

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= SpiderRails
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,32 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'SpiderRails'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/lib/spider_rails.rb

@@ -0,0 +1,4 @@
+# -*- encoding : utf-8 -*-
+current_file_name = __FILE__.split('/').last.gsub('.rb', '')
+Dir[File.expand_path("../#{current_file_name}/*.rb", __FILE__)].each { |file| require file }
+

data/lib/spider_rails/bilibili.rb

@@ -0,0 +1,68 @@
+module Spider
+  class BiliBili < Common
+
+    def get_res
+      animations = Hash.new
+      #browser for get page number
+      @b1 = goto("http://www.bilibili.tv/video/part-twoelement-1.html")
+      @b1.link(class: 'endPage').click
+      page_num = @b1.url[/\b\-\d+/].sub(/\-/, '')
+      @b1.close
+
+      #browser for get resources
+      browser2 = Watir::Browser.new :chrome
+      for i in 1..page_num.to_i
+        #for i in 1..1
+        browser2.goto "http://www.bilibili.tv/video/part-twoelement-#{i}.html"
+        html = Nokogiri::HTML.parse(browser2.html)
+
+        get_content(html, 'li.l2') do |li|
+          get_content(li, 'a.title') { |name| @name = name.content }
+
+          get_content(li, 'a.preview') { |a| @res = a['href'] }
+          get_content(li, 'a.preview img') { |img| @preview = img['src'] }
+
+          save_raw_data @name, @res, @preview
+        end
+      end
+
+      browser2.close
+      animations
+    end
+
+    alias_method :get_bilibili_res, :get_res
+
+
+    def goto url
+      browser1 = Watir::Browser.new :chrome
+      browser1.goto url
+      browser1
+    end
+
+    def save_raw_data(name, res, preview)
+      #anis.each do |preview, name|
+      #  unless AniRaw.where(preview: preview).exists?
+      #    AniRaw.create!(preview: preview, name: name)
+      #  end
+      #end
+
+      save_record(AniRaw, name: name, preview: preview, res: res)
+    end
+
+    #deprecated
+    #def save_record(model, *args)
+    #  if model.find_by_name(name) && !name.nil?
+    #    id = model.find_by_name(name).id
+    #    model.update(id, name: name, res: res, preview: preview)
+    #  else
+    #    model.create(name: name, res: res, preview: preview)
+    #  end
+    #end
+
+    def save_handled_data
+      AniRaw.all.each do |record|
+        save_record AniFin, name: record.name, preview: record.preview, res: record.res
+      end
+    end
+  end
+end

data/lib/spider_rails/common.rb

@@ -0,0 +1,62 @@
+module Spider
+  class Common
+    def full_site
+
+    end
+
+    def full_site_filter
+
+    end
+
+    def full_page
+      close_all_chromes
+    end
+
+    def get element
+
+    end
+
+    def single(element)
+
+    end
+
+    def single_filter
+    end
+
+    def get_content(element, selector, &block)
+      begin
+        if block_given?
+          element.css(selector).each &block
+        else
+          element.css(selector).each do |e|
+            if e.content != 0
+              #Just get first element
+              return e.content
+            end
+          end
+        end
+      end
+    end
+
+  end
+
+  class Browser
+    class << self
+      def open_browser url
+        @browser = Watir::Browser.new :chrome, switches: %w( --user-data-dir=/home/zxr/.config/google-chrome)
+        @browser.goto url
+        @browser
+      end
+
+      def close_all_chromes
+        chromes = `xdotool search 'google-chrome'`
+        chromes = chromes.split(/\s+/)
+        chromes.each do |chrome|
+          `xdotool windowkill #{chrome}`
+        end
+      end
+    end
+  end
+end
+
+

data/lib/spider_rails/google_dict.rb

@@ -0,0 +1,91 @@
+module Spider
+  class GoogleDict < Common
+    def get_cards(keywords)
+      Headless.new.start
+      keywords.each do |keyword|
+        unless Card.find_by_word(keyword)
+          get_card keyword
+          save_record(Card, word: @card[:Word],
+                      voice: @card[:Voice],
+                      verb: @card[:Verb],
+                      adj: @card[:Adjective],
+                      noun: @card[:Noun],
+                      pronoun: @card[:Pronoun],
+                      synonyms: @card[:Synonyms],
+                      abbr: @card[:Abbreviation],
+                      prep: @card[:Preposition],
+                      conj: @card[:Conjunction]
+          )
+        end
+      end
+    end
+
+    def get_keywords(path)
+      f = File.new(path)
+      dict = f.read.split(/\W/)
+      dict.delete("")
+      dict.uniq!
+      dict
+    end
+
+    class << self
+      def alias_methods(*args)
+        args.each do |arg|
+          alias_method arg, args.last
+        end
+      end
+    end
+
+    def get_card keyword
+      @card = Hash.new
+      @b = open_browser "https://www.google.com.hk/search?newwindow=1&safe=strict&q=#{keyword}+define&oq=#{keyword}+define"
+
+      doc = Nokogiri::HTML.parse @b.html
+
+      GoogleDict.alias_methods :card, :voice, :word, :get_content
+      card(doc, 'li.dct') do |c|
+        @card[:Word] = keyword.downcase
+        @card[:Voice] = voice(c, 'h3+.vk_sh')
+
+        # Get word explainations
+        get_explain(c)
+      end
+
+      @b.close
+      @card.delete(0)
+      @card
+    end
+
+    def get_explain(c)
+      type_nodes = c.css('div.vk_gy.vk_sh').to_a
+      content_nodes = c.css('div.vk_gy.vk_sh+div').to_a
+      type_nodes.each_with_index do |t, i|
+        table = content_nodes[i]
+        if table.css('li').count >= 2
+          fin_content = Array.new
+          table.css('li').each do |l|
+            fin_content << l.content
+          end
+        else
+          fin_content = table.content
+        end
+        @card[t.text.to_sym] = fin_content
+      end
+    end
+
+    #def login(username, password)
+    #  element?('a.gbgt#gb_70') { |e| e.click }
+    #  @b.text_field(name: 'Email').set username
+    #  @b.text_field(name: 'Passwd').set password
+    #  element?('input#signIn') { |e| e.click }
+    #end
+
+    def element?(selector, &block)
+      e = @b.element(css: selector)
+      if yield e
+      else
+        'element is nil'
+      end
+    end
+  end
+end

data/lib/spider_rails/hl.rb

@@ -0,0 +1,13 @@
+module Lib
+  module Hl
+    class << self
+      def run
+        h = Headless.new
+        h.start
+        b = Watir::Browser.new :chrome, switches: %w[--proxy-server=socks5://127.0.0.1:7070]
+        b.goto 'https://www.google.com.hk/search?q=google+define&oq=google+define'
+        p b.title
+      end
+    end
+  end
+end

data/lib/spider_rails/ji_ying.rb

@@ -0,0 +1,61 @@
+module Spider
+  class JiYing < Common
+    attr_accessor :ani
+
+    def initialize browser
+      @ani = Hash.new
+      @b = browser
+    end
+
+    def full_site
+      fp = final_page
+      begin
+        (1..fp).each do |page_num|
+          full_page page_num
+        end
+      rescue Exception
+        raise %Q(page isn't not exist)
+      end
+    end
+
+    def final_page
+      @b.element(css: '.title h2 a').text[/\(.+\)/].gsub!(/\(|\)/, '').to_i/100 + 1
+    end
+
+    def full_page page_num
+      @b.goto "http://bt.ktxp.com/search.php?keyword=%E8%AF%B8%E7%A5%9E&sort_id=28&field=title&order=&page=#{page_num}"
+      html = Nokogiri::HTML.parse @b.html
+      html.css('ltext').each do |td|
+        single(td)
+      end
+    end
+
+    def single element
+      single_filter
+      save Ani
+    end
+
+    def get(element)
+      element.css('a.quick-down').each do |a|
+        @ani[:torrent] = a['href']
+      end
+      element.css('a.quick-down+a').each do |a|
+        @ani[:title] = a.content
+      end
+      @ani
+    end
+
+    def save model_name
+      model_name.create(title: @ani[:title], torrent: @ani[:torrent])
+    end
+
+    def single_filter
+      unless @ani[:title][/外挂/]
+        @ani = nil
+      end
+
+      @ani[:torrent].prepend('http://bt.ktxp.com/')
+    end
+  end
+end
+

data/lib/spider_rails/sample_data.rb

@@ -0,0 +1,10 @@
+module Spider
+  class SampleData < Common
+    def generate
+      100.times do |n|
+        Novel.create(title: "やめて#{n}", content: 'やめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめて')
+        Card.create(word: "やめて#{n}", voice: 'やめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめてやめて やめて')
+      end
+    end
+  end
+end

data/lib/spider_rails/spread_sheet.rb

@@ -0,0 +1,32 @@
+module GoogleDrive
+  class SpreadSheet
+    def transfer
+      proxy = Net::HTTP.Proxy('127.0.0.1', 8087)
+      session = GoogleDrive.login("zhuxingruotest@gmail.com", "zhuxingruo", proxy)
+
+      ws = session.spreadsheet_by_key("0AiMMAt6U-_eEdFdNbXBfUjRMTlpsdV83OE9UWTRzTUE").worksheets[0]
+      #ws[2, 1] = "BakaBaka"
+      #ws[2, 2] = "Fuck you"
+      Card.columns.each_with_index do |c, i|
+        i = i + 1
+        ws[1, i] = c.name
+      end
+
+      Card.all.each_with_index do |card, i|
+        p "card.id:#{card.id}, i: #{i}"
+
+        if card.id >= 869
+          cf = card.attributes.each_with_index do |attr, j|
+            j = j + 1
+            #attr[0] is attr name, attr[1] is attr value
+            ws[i, j] = attr[1]
+          end
+
+          ws.save()
+          ws.reload()
+        end
+      end
+    end
+  end
+end
+

data/lib/spider_rails/version.rb

@@ -0,0 +1,3 @@
+module SpiderRails
+  VERSION = "0.0.1"
+end

data/test/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.