espider 0.5.8

data/lib/espider/front/qunar/rank.rb

@@ -0,0 +1,60 @@
+require 'nokogiri'
+require 'headless'
+require 'capybara'
+require 'capybara/dsl'
+module ESpider
+	module Front
+		module Qunar
+			class Rank < Base
+				include Capybara::DSL
+				def set_cookies
+					headless = Headless.new
+					headless.start
+					Capybara.current_driver = :selenium
+					Capybara.app_host = BASEURI
+					Capybara.default_wait_time = 10
+					begin
+						visit "/city/#{@cityurl}/dt-#{@hotelcode}/"
+					rescue Errno::ECONNREFUSED => e
+						raise
+					end
+					@cookies = {}
+					Capybara.current_session.driver.browser.manage.all_cookies.each { |cookie| @cookies[cookie[:name]] = cookie[:value] }
+				end
+				def load_price_ajax
+					mixKey = parse_mixkey
+					set_cookies
+					raise QunarMixKeyParsedException if mixKey.nil?
+					uri = URI(File.join(BASEURI, 'render', 'detailV2.jsp'))
+					params = {:fromDate=>Date.today+2, :toDate=>Date.today+3, :cityurl=>@cityurl, :HotelSEQ=>@HotelSEQ, :mixKey=>mixKey}
+					headers = Hash.new
+					headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 5.1; rv:26.0) Gecko/20100101 Firefox/26.0'
+					headers['Connection'] = 'keep-alive'
+					headers['Accept-Encoding'] = 'gzip, deflate'
+					headers['Cookie'] = @cookies.map{|k, v| k+'='+v}.join('; ')
+					uri.query = URI.encode_www_form(params)
+					res = HTTParty.get(uri.to_s, :headers=>headers)
+					json_ajax = parse_json_from_response_body(res.body)
+					if !json_ajax['errcode'].nil? and json_ajax['errcode'].eql?110
+						raise QunarIpBlockException,json_ajax
+					end
+					raise QunarIpBlockException,json_ajax if !json_ajax['errcode'].nil? and json_ajax['errcode'].eql?110
+					@ranks = json_ajax['result'].values
+				end
+				def rank(ota, room)
+					@ranks.each { |rank| return rank[15]+1 if rank[3].eql?room and rank[6].eql?ota and rank[9].eql?1 }
+					return nil
+				end
+				def rooms
+					@rooms ||= @ranks.map { |e|  e[3] }.uniq
+				end
+				def otas
+					@otas ||= @ranks.map{|e| e[6] }.uniq
+				end
+				def parse_json_from_response_body(res)
+					JSON.parse(res.gsub(/[()]/,""))
+				end
+			end
+		end
+	end
+end

data/lib/espider/front/ta.rb

@@ -0,0 +1,27 @@
+module ESpider
+	module Front
+		module TA
+			class TripAdvisor
+				attr_accessor	:base_url
+				def initialize(ta_id)
+					url = base_url+ta_id
+					res = HTTParty.get(url)
+					@code = res.code
+					@page = res.force_encoding("UTF-8")
+				end
+				def code
+					@code
+				end
+				def rank
+					ranks = []
+					@page.scan(/(Vendor\(.*\);)$/) do |ota|
+						ranks << ota[0].match(/\}\),\s*\".*\",/)[0].sub("}), \"","").sub("\",","")
+					end
+					ranks
+				end
+			end
+		end
+	end
+end
+require 'espider/front/ta/daodao'
+require 'espider/front/ta/advisor'

data/lib/espider/front/ta/advisor.rb

@@ -0,0 +1,12 @@
+module ESpider
+	module Front
+		module TA
+			class Advisor < TA::TripAdvisor
+				def initialize(ta_id)
+					@base_url = "http://www.tripadvisor.com/"
+					super(ta_id)
+				end
+			end
+		end
+	end
+end

data/lib/espider/front/ta/daodao.rb

@@ -0,0 +1,16 @@
+module ESpider
+	module Front
+		module TA
+			class Daodao < TA::TripAdvisor
+				def initialize(ta_id)
+					@base_url = "http://www.daodao.com/"
+					super(ta_id)
+				end
+				def rank
+					ranks = super
+					ranks[0...ranks.size/2]
+				end
+			end
+		end
+	end
+end

data/lib/espider/version.rb

@@ -0,0 +1,3 @@
+module ESpider
+  VERSION = "0.5.8"
+end

data/spec/advisor_front_rank_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+describe 'Advisor' do
+	it '正确实例化' do
+		ad = ESpider::Front::TA::Advisor.new('199326')
+		ad.should be_an_instance_of ESpider::Front::TA::Advisor
+	end
+	it '正确解析排名数据' do
+		ad = ESpider::Front::TA::Advisor.new('199326')
+		ad.rank.should be_an_instance_of Array
+	end
+end

data/spec/baidu_map_api_spec.rb

@@ -0,0 +1,31 @@
+#coding:utf-8
+require 'spec_helper'
+describe "百度地图api测试" do 
+	place = ESpider::Baidu::Map::API::Place.new('KNFShrAjm3BvgNEsNLfPSlAP')
+	it "should 被实例化" do
+		place.class.should == ESpider::Baidu::Map::API::Place
+	end
+	it "城市内检索请求" do
+		place.search_by_city('酒店', '北京')
+		place.next_page.class.should == String
+	end
+	it "矩形框检索请求" do
+		place.search_by_rect('酒店', '39.915,116.404,39.975,116.414')
+		place.next_page.class.should == String
+	end
+	it "圆形框检索请求" do
+		place.search_by_city('酒店', '北京')
+		place.next_page.class.should == String
+	end
+	it "http状态码为200" do
+		place.page_code.should == 200
+	end
+	it	"正确翻页" do
+		place.next_page.class.should == String
+		place.page_code.should == 200
+	end
+	it "获得查询结果总量" do 
+		place.search_by_city('酒店', '北京')
+		place.total.class.should == Fixnum
+	end
+end

data/spec/baidu_map_front_spec.rb

@@ -0,0 +1,53 @@
+#coding:utf-8
+require 'spec_helper'
+describe "百度地图酒店信息抓取测试" do 
+	hotel_list = ESpider::Front::Baidu::Map::Hotel::List.new('黔西南')
+	hotels = hotel_list.list
+	hotels_count = hotel_list.total
+	page_num = hotels_count/10
+	hotel = ESpider::Front::Baidu::Map::Hotel::Detail.new('1258371975449935871')
+	it "should 被实例化" do
+		hotel_list.should be_an_instance_of ESpider::Front::Baidu::Map::Hotel::List
+		hotel.should be_an_instance_of ESpider::Front::Baidu::Map::Hotel::Detail
+	end
+	it "should 正确解析酒店数量" do 
+		hotels_count.should be_an_instance_of Fixnum
+	end
+	it "should 正确返回酒店列表信息" do
+		hotels.should be_an_instance_of Array
+		hotels[0].should be_an_instance_of Hash
+	end
+	it "should 返回酒店名称" do
+		hotel.hotel_name.should be_an_instance_of String
+	end
+	it "should 返回酒店地址" do
+		hotel.hotel_addr.should be_an_instance_of String
+	end
+	it "should 返回酒店电话" do
+		hotel.hotel_tel.should be_an_instance_of String
+	end
+	it "should 返回酒店经纬度" do
+		hotel.hotel_geo.should be_an_instance_of Array
+	end
+	it "should 返回酒店星级" do
+		hotel.hotel_star.should be_an_instance_of String
+	end
+	it "should 返回酒店类别" do
+		hotel.hotel_category.should be_an_instance_of String
+	end
+	it "should 返回酒店价格" do
+		hotel.hotel_price.should be_an_instance_of String
+	end
+	it "should 返回酒店设施" do
+		hotel.hotel_facility.should be_an_instance_of String
+	end
+	it "should 返回酒店短评" do
+		hotel.hotel_short_comm.should be_an_instance_of String
+	end
+	it "should 返回酒店评论" do
+		hotel.hotel_review.should be_an_instance_of Array
+	end
+	it "should 返回酒店图片地址" do
+		hotel.hotel_image('669775d035aa1b42fd0eb008').should be_an_instance_of Array
+	end
+end

data/spec/dadao_front_rank_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+describe 'Daodao' do
+	it '正确实例化' do
+		dd = ESpider::Front::TA::Daodao.new('199326')
+		dd.should be_an_instance_of ESpider::Front::TA::Daodao
+	end
+	it '正确解析排名数据' do
+		dd = ESpider::Front::TA::Daodao.new('199326')
+		dd.rank.should be_an_instance_of Array
+	end
+end

data/spec/dianping_api_business_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+describe "大众点评api测试,business类api" do 
+	# it '正确实例化' do
+	# 	busi = ESpider::API::Dianping::Business.new('16935220','0c3d163c8fff4639a3372aea71df52a1')
+	# 	busi.class.should == ESpider::API::Dianping::Business
+	# end
+	# it '正确调用find_businesses接口,传入经纬度信息' do
+	# 	busi = ESpider::API::Dianping::Business.new('16935220','0c3d163c8fff4639a3372aea71df52a1')
+	# 	busi.find_businesses('上海','','31.18268013000488','121.42769622802734').should == true
+	# 	busi.get.class.should == String
+	# end
+	# it '正确调用find_businesses接口,并翻页' do
+	# 	busi = ESpider::API::Dianping::Business.new('16935220','0c3d163c8fff4639a3372aea71df52a1')
+	# 	busi.find_businesses('上海','','31.18268013000488','121.42769622802734').should == true
+	# 	busi.get.class.should == String
+	# 	p busi.get
+	# 	busi.next_page.class.should == String
+	# end
+	# it '正确调用find_businesses接口，加入分类信息' do
+	# 	busi = ESpider::API::Dianping::Business.new('16935220','0c3d163c8fff4639a3372aea71df52a1')
+	# 	busi.find_businesses('上海','','31.18268013000488','121.42769622802734').should == true
+	# 	busi.set_category('酒店')
+	# 	busi.next_page.class.should == String
+	# end
+	it '正确调用/get_single_business接口，返回单个商户信息' do
+		busi = ESpider::API::Dianping::Business.new('16935220','0c3d163c8fff4639a3372aea71df52a1')
+		busi.get_single_business('8733744').should be_instance_of String
+	end
+end

data/spec/dianping_api_metadata_spec.rb

@@ -0,0 +1,44 @@
+#coding:utf-8
+require 'spec_helper'
+describe "大众点评api测试,metadata类api" do 
+	mdata = ESpider::API::Dianping::Metadata.new('16935220','0c3d163c8fff4639a3372aea71df52a1')
+	it '正确实例化' do
+		mdata.class.should == ESpider::API::Dianping::Metadata
+	end
+	it '获取支持商户搜索的最新城市列表' do
+		mdata.get_cities_with_businesses
+		mdata.get.class.should == String
+	end
+	it '获取支持商户搜索的最新城市下属区域列表 ' do
+		mdata.get_regions_with_businesses
+		mdata.get.class.should == String
+	end
+	it '获取支持商户搜索的最新分类列表 ' do
+		mdata.get_categories_with_businesses
+		mdata.get.class.should == String
+	end
+	it '获取支持团购搜索的最新城市列表 ' do
+		mdata.get_cities_with_deals
+		mdata.get.class.should == String
+	end
+	it '获取支持团购搜索的最新城市下属区域列表 ' do
+		mdata.get_regions_with_deals
+		mdata.get.class.should == String
+	end
+	it '获取支持团购搜索的最新分类列表 ' do
+		mdata.get_categories_with_deals
+		mdata.get.class.should == String
+	end
+	it '获取支持优惠券搜索的最新城市列表 ' do
+		mdata.get_cities_with_coupons
+		mdata.get.class.should == String
+	end
+	it '获取支持优惠券搜索的最新城市下属区域列表 ' do
+		mdata.get_regions_with_coupons
+		mdata.get.class.should == String
+	end
+	it '获取支持优惠券搜索的最新分类列表 ' do
+		mdata.get_categories_with_coupons
+		mdata.get.class.should == String
+	end
+end

data/spec/dianping_front_detail_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+describe "ESpider::Front::Dianping::Hotel::Detail" do
+	it '实例化' do
+		detail = ESpider::Front::Dianping::Hotel::Detail.new('2136037')
+		detail.should be_instance_of ESpider::Front::Dianping::Hotel::Detail
+	end
+	it '返回酒店名称' do
+		detail = ESpider::Front::Dianping::Hotel::Detail.new('2136037')
+		detail.name.should be_instance_of String
+	end
+	it '返回酒店描述' do
+		detail = ESpider::Front::Dianping::Hotel::Detail.new('2869420')
+		detail.intro.should be_instance_of String
+	end
+	it '返回酒店设施' do
+		detail = ESpider::Front::Dianping::Hotel::Detail.new('2869420')
+		detail.facilities.should be_instance_of Array
+	end
+	it '返回酒店图片' do
+		detail = ESpider::Front::Dianping::Hotel::Detail.new('2869420')
+		detail.images.should be_instance_of Array
+	end
+	it '返回酒店评论' do
+		detail = ESpider::Front::Dianping::Hotel::Detail.new('2869420')
+		detail.comments.should be_instance_of Array
+	end
+end

data/spec/dianping_front_list_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+describe "ESpider::Front::Dianping::Hotel::List" do
+	list = ESpider::Front::Dianping::Hotel::List.new('1')
+	it '实例化' do
+		list.should be_instance_of ESpider::Front::Dianping::Hotel::List
+	end
+	it '返回名称' do
+		list.should be_instance_of ESpider::Front::Dianping::Hotel::List
+	end
+	it '返回酒店列表' do
+		list.hotels(1).should be_instance_of Array
+	end
+	it '正确计算酒店数量' do
+		list = ESpider::Front::Dianping::Hotel::List.new('2','1489')
+		list.total.should be_instance_of Fixnum
+	end
+end

data/spec/kuxun_front_detail_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+describe "ESpider::Front::Kuxun::Detail" do
+	detail = ESpider::Front::Kuxun::Detail.new('jiarijiudian-haerbin-hotel_1')
+	it '实例化' do
+		detail.should be_instance_of ESpider::Front::Kuxun::Detail
+	end
+	it '返回酒店名称' do
+		detail.name.should be_instance_of String
+	end
+	it '返回酒店地址' do
+		detail.address.should be_instance_of String
+	end
+	it '返回酒店描述' do
+		detail.intro.should be_instance_of String
+	end
+	it '返回酒店设施' do
+		detail.facilities.should be_instance_of Array
+	end
+	it '返回酒店图片' do
+		detail.images.should be_instance_of Array
+	end
+	it '返回酒店经纬度' do
+		detail.xy.should be_instance_of Array
+		detail.xy.size.should == 2
+	end
+end

data/spec/qunar_front_detail_spec.rb

@@ -0,0 +1,74 @@
+#coding: UTF-8
+require 'spec_helper'
+describe "qunar信息抓取测试" do
+	qunar_id = "beijing_city_2807"
+	city = "beijing_city"
+	hotel = ESpider::Front::Qunar::Hotel.new(qunar_id)
+	it 'should 被正确实例化' do
+		hotel.should be_an_instance_of ESpider::Front::Qunar::Hotel
+	end
+	it 'should 返回酒店名称' do
+		hotel.name.should be_an_instance_of String
+	end
+	it 'should 返回正确的酒店星级' do
+		hotel.star.should >= 0
+		hotel.star.should <= 5
+	end
+	it 'should 返回qunar酒店城市英文名' do
+		hotel.citynameEn.should == 'beijing'
+	end
+	it 'should 返回qunar酒店地址' do
+		hotel.address.should be_an_instance_of String
+	end
+	it 'should 返回qunar酒店电话' do
+		hotel.phone.should be_an_instance_of String
+	end
+	# it 'should 返回qunar酒店描述' do
+	# 	hotel.desc.class.should == String 
+	# end
+	# it 'should 返回qunar酒店开业日期' do
+	# 	hotel.insttime.should == '2008年'
+	# end
+	# it 'should 返回qunar酒店概述' do
+	# 	hotel.abstract_desc.should == '位于北京市东城区菊儿胡同、紧邻北二环主干线。网友评价说“小吃很多也便宜”。'
+	# end
+	# it 'should 返回qunar酒店品牌' do
+	# 	hotel.brand.should == '汉庭酒店'
+	# end
+	# it 'should 返回qunar酒店参考价格' do
+	# 	hotel.reference_price.should == '120'
+	# end
+	# it 'should 返回qunar酒店google经纬度' do
+	# 	hotel.xy.should == ["39.939323", "116.4054"]
+	# end
+	# it 'should 返回qunar酒店设施' do
+	# 	hotel.facilities.should == [{"name"=>"房间设施", "info"=>["宽带上网", "空调", "暖气", "24小时热水", "吹风机", "国际长途电话"]}, {"name"=>"酒店服务", "info"=>["接待外宾", "叫醒服务", "行李寄存", "洗衣服务", "租车", "早餐服务"]}, {"name"=>"酒店设施", "info"=>["无线上网公共区域", "停车场", "无烟房", "中式餐厅", "会议室", "商务中心"]}]
+	# end
+	# it 'should 返回qunar交通信息' do
+	# 	hotel.traffic.should == [{"name"=>"战鼓楼", "distance"=>"594.056489643499"}, {"name"=>"蓬蒿剧场", "distance"=>"395.18496791364"}, {"name"=>"中央戏剧学院实验剧场", "distance"=>"425.239611398247"}, {"name"=>"和中堂足道(交道口店)", "distance"=>"488.729152256305"}, {"name"=>"周末相声俱乐部", "distance"=>"524.822582351895"}, {"name"=>"鲸鱼桌游吧", "distance"=>"790.623187595583"}, {"name"=>"一来二去桌游主题休闲吧", "distance"=>"988.900711965127"}, {"name"=>"东城区文化馆", "distance"=>"606.324700795644"}, {"name"=>"国话小剧场", "distance"=>"757.099876935085"}, {"name"=>"东城区图书馆", "distance"=>"661.53698062375"}]
+	# end
+	# it 'should 返回qunar酒店最后装修时间' do
+	# 	hotel.decorate_date.should == ''
+	# end
+	# it 'should 返回qunar酒店评论' do
+	# 	hotel.comments.should == ''
+	# end
+	# it 'should 返回qunar酒店评论总数' do
+	# 	hotel.total_comment.should_not = '339'
+	# end
+	# it 'should 返回qunar酒店评论好评数' do
+	# 	hotel.good_comment.should_not = '250'
+	# end
+	# it 'should 返回qunar酒店图片' do
+	# 	hotel.images.should == ''
+	# end
+	# it 'should 返回qunar酒店评分' do
+	# 	hotel.score.to_f.should < 10.0
+	# end
+	# it 'should 返回qunar酒店城市中文名' do
+	# 	hotel.citynameCn.should == '北京'
+	# end
+	# it 'should 返回qunar酒店城市英文名' do
+	# 	hotel.citynameEn.should == 'beijing'
+	# end
+end