rails_spider 0.1.0

data/lib/sync_qiniu.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'fileutils'
+require 'open-uri'
+require 'bundler'
+require 'logger'
+Bundler.require
+
+Mongoid.load!("./config/mongoid.yml","development")
+
+Dir.glob("#{File.dirname(__FILE__)}/models/*.rb").each do |m|
+  require m
+end
+
+FileUtils.mkdir_p("log")
+logger = Logger.new("log/getimage_log.txt")
+
+@events = Event
+
+logger.log(1, 'Prepare folder')
+dir = "images/faces/#{Date.today.to_s.gsub("-", "_")}"
+log =  "Try to make dir #{dir}"
+logger.log(1, log)
+FileUtils.mkdir_p(dir)
+
+pwd = File.dirname(File.expand_path('.', __FILE__))
+log =  "pwd #{pwd}"
+logger.log(1, log)
+config = "#{pwd}/config/qiniu_conf.json"
+qrsync = "#{pwd}/vendor/qiniu/qrsync"
+log = 'sync to qiniu after download all images.'
+logger.log(1, log)
+cmd = "#{qrsync} #{config}"
+logger.log(1, cmd)
+`#{cmd}`

data/lib/sync_qiniu/getimages.rb

@@ -0,0 +1,98 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'fileutils'
+require 'open-uri'
+require 'bundler'
+require 'logger'
+Bundler.require
+
+Mongoid.load!("./config/mongoid.yml","development")
+
+Dir.glob("#{File.dirname(__FILE__)}/models/*.rb").each do |m|
+  require m
+end
+
+Dir.glob("#{File.dirname(__FILE__)}/helper/*.rb").each do |h|
+  require h
+end
+
+FileUtils.mkdir_p("log")
+logger = Logger.new("log/getimage_log.txt")
+
+@events = Event
+
+logger.log(1, 'Prepare folder')
+dir = "images/faces/#{Date.today.to_s.gsub("-", "_")}"
+log =  "Try to make dir #{dir}"
+logger.log(1, log)
+FileUtils.mkdir_p(dir)
+
+@events = @events.where(filename: nil)
+
+count_to_download = @events.count
+
+log = "Will download #{count_to_download} events face"
+logger.log(1, log)
+
+@events = @events.to_a
+
+group_len = 800
+
+threads = []
+
+@events.each_slice(group_len).each do |event_group|
+  threads << Thread.new do
+    event_group.each do |e|
+      begin
+        data = open(URI.encode(e.face)){ |f| f.read }
+
+        filetype = File.extname(e.face).gsub(/\?.*$/, "")
+        filename = "face-" + e.id.to_s + filetype
+        filename = "#{dir}/#{filename}"
+        open(filename, "wb") { |f| f.write(data) }
+        log = "Download #{filename} to images/..."
+        logger.log(1, log)
+        e.update_attribute(:filename, filename)
+        sleep 0.5
+      rescue => se
+        log = "throw a exception:\n#{se.message}\n#{se.backtrace}\n#{e.url}"
+        logger.log(1, log)
+        sleep 1
+      end
+    end
+  end
+end
+
+threads.map(&:join)
+
+# sync to qiniu after download all images.
+if count_to_download > 0
+  pwd = File.dirname(File.expand_path('.', __FILE__))
+  log =  "pwd #{pwd}"
+  logger.log(1, log)
+  config = "#{pwd}/config/qiniu_conf.json"
+  qrsync = "#{pwd}/vendor/qiniu/qrsync"
+  log = 'sync to qiniu after download all images.'
+  logger.log(1, log)
+  cmd = "#{qrsync} #{config}"
+  logger.log(1, cmd)
+  `#{cmd}`
+end
+
+threads = []
+@events = Event.where(poster_width: nil).to_a
+@events.each_slice(group_len).each do |event_group|
+  threads << Thread.new do
+    event_group.each do |e|
+      next if e.filename.blank?
+      begin
+        e.update_image_info
+      rescue  => se
+        log = "#{Time.now}\n#{se.message}\n#{se.backtrace}#{e.url}"
+        logger.log(1, log)
+      end
+    end
+  end
+end
+
+threads.map(&:join)

data/lib/sync_qiniu/getimages_info.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'fileutils'
+require 'open-uri'
+require 'bundler'
+Bundler.require
+
+Mongoid.load!("./config/mongoid.yml","development")
+
+Dir.glob("#{File.dirname(__FILE__)}/models/*.rb").each do |m|
+  require m
+end
+
+group_len = 200
+id = "534274765275627a63000000"
+count = Event.where(poster_width: nil, :id.gt => id).count
+while count > 100
+  threads = []
+  @events = Event.where(poster_width: nil, :id.gt => id).limit(1000).to_a
+  @events.each_slice(group_len).each do |event_group|
+    threads << Thread.new do
+      event_group.each do |e|
+        next if e.filename.blank?
+        begin
+          e.update_image_info
+        rescue  => se
+          log = "#{Time.now}\n#{se.message}\n#{se.backtrace}#{e.url}"
+          puts log
+        end
+      end
+    end
+  end
+  threads.map(&:join)
+  id = @events.last.id.to_s
+  count = Event.where(poster_width: nil, :id.gt => id).count
+end
+

data/lib/sync_qiniu/getlocation.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'bundler'
+require 'net/http'
+Bundler.require
+
+Mongoid.load!("./config/mongoid.yml","development")
+
+Dir.glob("#{File.dirname(__FILE__)}/models/*.rb").each do |m|
+  require m
+end
+
+@events = Event.where(:source.ne => "douban") # can not use only because of update attribute
+
+@events.each do |e|
+  puts "ID: #{e.id}"
+  puts e.place
+  url = URI.escape("http://api.map.baidu.com/geocoder/v2/?address=#{e.place}&output=json&ak=A38d59da730152d77b407446a3c0dd2b")
+  # Geocoding API:  http://developer.baidu.com/map/webservice-geocoding.htm
+  # http://api.map.baidu.com/geocoder/v2/?address=%E5%BE%90%E5%AE%B6%E6%B1%87&output=json&ak=A38d59da730152d77b407446a3c0dd2b&callback=showLocation
+  begin
+    response = Net::HTTP.get_response(URI(url))
+    puts response.body
+    data = response.body
+  rescue SocketError
+    sleep 10
+    retry
+  end
+    result = JSON.parse(data)
+    puts result
+  # status 说明文档:  http://developer.baidu.com/map/webservice-geocoding.htm#.E6.8E.A5.E5.8F.A3.E7.A4.BA.E4.BE.8A
+  if result["status"] != 0
+    e.location = [0.0, 0.0]
+    e.save
+    puts "This place can not translate to location......"
+  else
+    # {"status"=>0, "result"=>{"location"=>{"lng"=>121.48026424818, "lat"=>31.229092805768}, "precise"=>1, "confidence"=>80, "level"=>"道路"}}
+    puts result["result"]["location"]
+    store_result = []
+    store_result << result["result"]["location"]["lng"]
+    store_result << result["result"]["location"]["lat"]
+    puts store_result
+    e.location = store_result
+    e.save
+    puts "save location #{store_result} of ID: #{e.id} success!"
+    puts '--------------------'
+  end
+end

data/lib/sync_qiniu/getproxy.rb

@@ -0,0 +1,95 @@
+# encoding: utf-8
+require 'rubygems'
+require 'bundler'
+Bundler.require
+
+# Fetch proxy from: http://proxy.com.ru/
+class ProxyList
+
+  def initialize
+    @proxylists = []
+    @urls = []
+    (1..2).each do |i|
+      @urls << "http://proxy.com.ru/list_#{i}.html"
+    end
+  end
+
+  def fetch_list
+    @urls.each do |url|
+		puts proxys_in_url(url).class
+      @proxylists += proxys_in_url(url)
+    end
+    @proxylists.uniq!
+    puts @proxylists
+    puts "Fetched proxys #{@proxylists.size}"
+  end
+
+  alias :start :fetch_list
+
+  def proxys_in_url(url)
+    proxys = []
+    agent = Mechanize.new
+    page = agent.get(url)
+  	list = page.search("body font table tr td:last font table tr")
+  	list[1..-1].each do |tr|
+  		ip = tr.search("td")[1].text
+  		port = tr.search("td")[2].text
+  		proxy = { ip: ip, port: port }
+  		proxy = verify_proxy(proxy) # 比较费时间
+  		proxys << proxy unless proxy == false
+  		puts "add #{proxy} to proxys array..."
+  	end 
+  	return proxys
+  end
+
+  def get_urls
+	  return @urls
+  end
+  
+  def get_proxylists
+	  return @proxylists
+  end
+
+  def save_proxylists
+	pls = @proxylists.to_s
+	puts pls
+	if File.exist?("proxylists.txt")
+	  File.rename("proxylists.txt","proxylists.txt.bak")
+	end
+	f = File.new("proxylists.txt","w+")
+	f.write(pls)
+	f.close
+	puts "Save proxylists successfully."
+  end
+
+  def save_all_proxylists
+	pls = @proxylists.to_s
+	puts pls
+	if File.exist?("allproxylists.txt")
+	  File.rename("allproxylists.txt","allproxylists.txt.bak")
+	end
+	f = File.new("allproxylists.txt","w+")
+	f.write(pls)
+	f.close
+	puts "Save all proxylists successfully."
+  end
+
+  def verify_proxy(proxy)
+	ip, port = proxy[:ip], proxy[:port]
+	testagent = Mechanize.new
+	testagent.set_proxy ip, port
+	testagent.read_timeout = 10
+	begin
+		page = testagent.get("http://www.baidu.com")	
+		if page.title == "百度一下，你就知道"
+		  return proxy
+		puts 'That a good proxy'
+		end
+	rescue => e
+		puts e.message
+		puts 'That a bad proxy'
+		proxy = nil
+		return
+	end
+  end
+end

data/lib/tasks/the_spider_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :rails_spider do
+#   # Task goes here
+# end

data/rakefile

@@ -0,0 +1,284 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'RailsSpider'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+load 'rails/tasks/statistics.rake'
+
+
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+# coding: utf-8
+#!/usr/bin/env ruby
+
+#require 'rubygems'
+#require 'fileutils'
+#require 'open-uri'
+#require 'bundler'
+
+ENV['SPIDER_ENV'] = 'development'
+require File.expand_path './config/environment'
+
+include EventSpider
+
+namespace :first do
+
+  desc 'grab all event of a city for the first time'
+  task :get_all_events do
+    spiders = [Douban]#, Weibo, Waitan, CityMoment]
+    spiders.each do |spider|
+      spider.new.run
+    end
+  end
+
+  # TODO 将不能使用的代理IP删除
+  desc 'get proxy from web'
+  task :test_proxy do
+    proxy_list = EventSpider::Proxy.proxy_list
+    puts proxy_list
+    puts "Proxy IP count: #{proxy_list.size}"
+    pro = EventSpider::Proxy.new
+    proxy_list.each do |proxy|
+      #agent.set_proxy proxy[:ip], proxy[:port]
+      puts "Test proxy IP:#{proxy}"
+      page = pro.get_page_from('http://www.douban.com', proxy, nil, 0)
+      if page
+        puts page.title
+      else
+        puts "Not get page by IP: #{proxy}"
+      end
+    end
+  end
+
+  desc 'Event sequid'
+  task :set_sequid do
+    es = Event.where(sequid: nil)#.order_by(:created_at.asc)
+    #total_count = es.count
+    #limit_count = 100
+    #page = total_count / limit_count
+    #page += 1 if (total_count % limit_count) > 0
+    i = Event.max(:sequid).to_i + 1
+    #(0..page).each do |pg|
+    es.to_a.each do |e|
+      e.update_attribute(:sequid, i)
+      puts "#{e.created_at} | #{e.int_id} |  #{e.sequid}"
+      i += 1
+    end
+    #end
+  end
+
+  desc 'Reset Event sequid number'
+  task :reset_event_sequid do
+    max = Event.max(:sequid)
+    puts "Event sequid max number is #{max}"
+    inc=MongoidAutoInc::Incrementor.new({})
+    res = inc['event_sequid'].set max
+    puts "Set the sequid to the max #{res}"
+  end
+
+end
+
+namespace :everyday do
+  desc 'Update grab and Sync image to Qiniu'
+  task :update_and_sync do
+    Rake::Task["everyday:grab_update"].invoke
+    Rake::Task["everyday:event_image"].invoke
+  end
+
+  desc 'update grab every day'
+  task :grab_update do
+    spiders = [Douban, Weibo, Waitan, CityMoment]
+    spiders.each do |spider|
+      spider.new.grab_update
+    end
+  end
+
+  desc 'download event image and sync to qiniu'
+  task :event_image do
+    logger = Logger.new("log/image.log")
+    images_dir = "images/faces/#{Date.today.to_s.gsub("-", "_")}"
+    FileUtils.mkdir_p(images_dir)
+    logger.info "Make image file folder #{images_dir}"
+
+    events = Event.where(filename: nil).to_a
+    logger.info "There are #{events.count} images need to download"
+    group_len = 800
+    threads = []
+
+    events.each_slice(group_len).each do |event_group|
+      threads << Thread.new do
+        event_group.each do |ent|
+          begin
+            image_data = open(URI.encode(ent.face)){|f| f.read}
+            filetype = File.extname(ent.face).gsub(/\?.*$/, '')
+            filename = "#{images_dir}/face-#{ent.id.to_s}#{filetype}"
+            logger.info "File type:#{filetype} | name:#{filename}"
+            open(filename, "wb") { |f| f.write(image_data) }
+            logger.info "Download #{filename}"
+            ent.update_attribute(:filename, filename)
+            sleep 1
+          rescue => e
+            logger.error e.message
+            # e.backtrace.each do |msg|
+            #   logger.error msg
+            # end
+            sleep 1
+          end
+        end
+      end
+    end
+    threads.map(&:join)
+
+    if events.count > 0
+      pwd = File.dirname(File.expand_path('.', __FILE__))
+      qiniu_config = "#{pwd}/config/qiniu_conf.json"
+      qiniu_sync = "#{pwd}/vendor/qiniu/qrsync"
+      cmd = "#{qiniu_sync} #{qiniu_config}"
+      logger.info "Sync images to QiNiu"
+      `#{cmd}`
+      logger.info "#{cmd} done"
+    end
+
+    threads = []
+    events = Event.where(poster_width: nil).to_a
+    events.each_slice(group_len).each do |event_group|
+      threads << Thread.new do
+        event_group.each do |ent|
+          next if ent.filename.blank?
+          begin
+            ent.update_image_info
+            logger.info "Update image info success with event:#{ent.id}"
+          rescue => e
+            logger.error e.message
+            #e.backtrace.each{|msg| logger.error msg}
+            logger.error "Event update image info wrong:#{ent.url}"
+          end
+        end
+      end
+    end
+    threads.map(&:join)
+  end
+
+  desc 'upate iamge info'
+  task :update_image_info do
+    logger = Logger.new("#{ENV['SPIDER_ENV']}.log")
+    threads = []
+    group_len = 800
+    events = Event.where(poster_width: nil).to_a
+    logger.info 'Start update image info'
+    events.each_slice(group_len).each do |event_group|
+      threads << Thread.new do
+        event_group.each do |ent|
+          next if ent.filename.blank?
+          begin
+            ent.update_image_info
+            logger.info "Update image info success with event:#{ent.id}"
+          rescue => e
+            logger.error e.message
+            #e.backtrace.each{|msg| logger.error msg}
+            logger.error "Event update image info wrong:#{ent.url}"
+          end
+        end
+      end
+    end
+    threads.map(&:join)
+    logger.info 'End of update image info'
+  end
+
+end
+
+namespace :proxy do
+  # 爬取www.youdaili.com上的代理IP
+  # 并校验是否能用
+  # 并保存到文件proxylists.txt中
+  desc 'get youdaili proxy IP'
+  task :proxy_youdaili do
+    used_proxy = EventSpider::Proxy.proxy_list.uniq
+    proxy_spider = EventSpider::Proxy.new(used_proxy)
+    proxy_list = proxy_spider.get_youdaili_proxy
+    proxy_list = proxy_spider.validate_proxy(proxy_list)
+    proxy_spider.write_proxy_to_file(proxy_list)
+  end
+
+  # http://www.xici.net.co/nn/
+  # 抓取代理IP，并校验，再保存到本地文件
+  desc 'get xici.net proxy IP'
+  task :proxy_xici do
+    used_proxy = EventSpider::Proxy.proxy_list.uniq
+    proxy_spider = EventSpider::Proxy.new(used_proxy)
+    proxys = proxy_spider.get_xici_proxy
+    proxys = proxy_spider.validate_proxy(proxys)
+    proxy_spider.write_proxy_to_file(proxys)
+  end
+
+  # 校验本地手动输入代理文件allproxylists.txt
+  desc 'validate allproxylists.txt proxy IP and save to proxylists.txt'
+  task :validate_allproxylists do
+    proxy_list = EventSpider::Proxy.get_allproxylists.uniq
+    proxy_spider = EventSpider::Proxy.new
+    proxys = proxy_spider.validate_proxy(proxy_list)
+    proxy_spider.write_proxy_to_file(proxys)
+  end
+
+
+  # http://api.map.baidu.com/geocoder/v2/?address=%20%E9%87%8D%E5%BA%86%20%E6%B2%99%E5%9D%AA%E5%9D%9D%E5%8C%BA%20NUTS&output=json&ak=A38d59da730152d77b407446a3c0dd2b&callback=showLocation
+  desc 're-get event location from baidu with location is [0.1,0.1]'
+  task :reget_location do
+    Event.where(location: [0.1, 0.1]).each do |event|
+      next if event.place.nil? || event.place == ''
+      baidu_api = URI.escape("http://api.map.baidu.com/geocoder/v2/?address=#{event.place}&output=json&ak=A38d59da730152d77b407446a3c0dd2b")
+      puts baidu_api
+      begin
+        response = Net::HTTP.get_response(URI(baidu_api))
+        data = response.body
+        result = JSON.parse(data)
+        if result["status"] != 0
+          location = [0.0, 0.0]
+        else
+          location = []
+          location << result["result"]["location"]["lng"]
+          location << result["result"]["location"]["lat"]
+        end
+        puts location
+      rescue Timeout::Error
+        i ||= 0
+        if i <= 5
+          retry
+        end
+      rescue => e
+        puts event.place
+        puts e
+        location = [0.1, 0.1]
+      end
+    end
+  end
+
+end
+
+
+
+task default: :test