espider 0.5.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 09e1fd711323b14fe56bd4acc1c3e8c4f451db93
+  data.tar.gz: 4975bc93cd785fb82abe3a79edb2a8be5a2406a1
+SHA512:
+  metadata.gz: ba15746f81e8658df9cf4dee7fe9aa7b949c085be8325af393260520bc79f5603376fabad5732412bc8adc4f17e52926291573155c501fc98e11d0c1864ba217
+  data.tar.gz: 06957b4ca9760e8834649affd3518687fb2385dec5fe8c5283af9895213f9593f596d0858a8cb3e1da0a59c06dfe5dbdb5e7bf52bd245922290e8ea29fb893ad

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in espider.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Espider
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'espider'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install espider
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/espider.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'espider/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "espider"
+  spec.version       = ESpider::VERSION
+  spec.authors       = ["lei"]
+  spec.email         = ["littlell@126.com"]
+  spec.description   = %q{elong spider}
+  spec.summary       = %q{elong spider for qunar,dianyping etc.}
+  spec.homepage      = "http://ruby-china.org/littlell"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", "~> 0"
+
+  # spec.add_dependency "date", "~> 0"
+  spec.add_dependency "nokogiri", "~> 0"
+  spec.add_dependency "httparty", "~> 0"
+  spec.add_dependency "json", "~> 0"
+end

data/lib/espider.rb

@@ -0,0 +1,10 @@
+require 'espider/version'
+require 'espider/front/qunar'
+require 'espider/front/front'
+require 'espider/front/dianping/hotel'
+require 'espider/front/baidu/map/hotel'
+require 'espider/front/ta'
+require 'espider/api/baidu/map'
+require 'espider/api/dianping'
+require 'espider/front/kuxun'
+require 'espider/exceptions'

data/lib/espider/api/baidu/map.rb

@@ -0,0 +1,9 @@
+require 'espider/api/baidu/map/place'
+module Espider
+	module API
+		module Baidu
+			module Map
+			end
+		end
+	end
+end

data/lib/espider/api/baidu/map/place.rb

@@ -0,0 +1,53 @@
+require 'nokogiri'
+require 'httparty'
+require 'uri'
+module ESpider
+	module Baidu
+		module Map
+			module API
+				class Place
+					FORMAT = {:JSON=>'json',:XML=>'xml'}
+					INFO = {:ALL=>2,:BASIC=>1}
+					FIRST_PAGE_NUM = -1
+					BaseUri = "http://api.map.baidu.com/place/v2/search?"
+					def initialize(key,page_size=10,info_type=INFO[:ALL],format=FORMAT[:JSON])
+						@key = key
+						@page_size = page_size
+						@info_type = info_type
+						@format = format
+					end
+					#城市内检索
+					# http://api.map.baidu.com/place/v2/search?ak=您的密钥&output=json&query=酒店&page_size=10&page_num=5&scope=2&region=北京
+					def search_by_city(query,region)
+						@params = {:ak=>@key,:output=>@format,:query=>query,:page_size=>@page_size,:page_num=>FIRST_PAGE_NUM,:scope=>@info_type,:region=>region}
+					end
+					#矩形区域搜索
+					#http://api.map.baidu.com/place/v2/search?ak=您的密钥&output=json&query=酒店&page_size=10&page_num=5&scope=2&bounds=39.915,116.404,39.975,116.414
+					def search_by_rect(query,bounds)
+						@params = {:ak=>@key,:output=>@format,:query=>query,:page_size=>@page_size,:page_num=>FIRST_PAGE_NUM,:scope=>@info_type,:bounds=>bounds}
+					end
+					#圆形区域检索
+					# http://api.map.baidu.com/place/v2/search?ak=您的密钥&output=json&query=酒店&page_size=10&page_num=5&scope=2&location=39.915,116.404&radius=2000
+					def search_by_round(query,location,radius)
+						@params = {:ak=>@key,:output=>@format,:query=>query,:page_size=>@page_size,:page_num=>FIRST_PAGE_NUM,:scope=>@info_type,:location=>location,:radius=>radius}
+					end
+					def next_page
+						@params[:page_num] = @params[:page_num] + 1
+						get_page
+					end
+					def get_page
+						@page = HTTParty.get File.join(BaseUri + URI.escape(@params.collect{|k,v| "#{k}=#{v}"}.join('&')))
+						@page.body
+					end
+					def page_code
+						@page.code
+					end
+					def total
+						parsed_json = JSON.parse @page.body
+						parsed_json['total']
+					end
+				end
+			end
+		end
+	end
+end

data/lib/espider/api/dianping.rb

@@ -0,0 +1,13 @@
+module ESpider
+	module API
+		module Dianping
+		end
+	end
+end
+require 'espider/api/dianping/base'
+require 'espider/api/dianping/review'
+require 'espider/api/dianping/metadata'
+require 'espider/api/dianping/deal'
+require 'espider/api/dianping/coupon'
+require 'espider/api/dianping/business'
+require 'espider/api/dianping/params'

data/lib/espider/api/dianping/base.rb

@@ -0,0 +1,35 @@
+require 'digest/sha1'
+require 'httparty'
+require 'uri'
+module ESpider
+	module API
+		module Dianping
+			class Base
+				Domain = "http://api.dianping.com"
+				Version = "v1"
+				def initialize(appkey,app_secret)
+					@appkey = appkey
+					@app_secret = app_secret
+					@params = {}
+				end
+				def calc_digest
+					@params.delete(:appkey)
+					@params.delete(:sign)
+					digest = Digest::SHA1.hexdigest(@appkey+@params.sort.map{|e| e[0].to_s+e[1].to_s}.join('')+@app_secret).upcase
+					@params[:appkey] = @appkey
+					@params[:sign] = digest
+				end
+				def get
+					calc_digest
+					@uri.query = URI.encode_www_form(@params)
+					res = HTTParty.get @uri.to_s
+					res.body
+				end
+				def next_page
+					@params[:page] = @params[:page] + 1
+					get
+				end
+			end
+		end
+	end
+end

data/lib/espider/api/dianping/business.rb

@@ -0,0 +1,38 @@
+require 'active_support/core_ext/string/inflections'
+module ESpider
+	module API
+		module Dianping
+			class Business < ESpider::API::Dianping::Base
+				BaseUri = File.join(Domain,Version,self.name.demodulize.downcase)
+				include ESpider::API::Dianping
+				def find_businesses(city=Param::EMPTY,region=Param::EMPTY,latitude=Param::EMPTY,longitude=Param::EMPTY)
+					return false if city.empty? and (longitude.empty? or latitude.empty?)
+					return false if city.empty? and !region.empty?
+					@uri = URI File.join(BaseUri,__method__.to_s)
+					method(__method__).parameters.each do |e| 
+						param_value = eval(e[1].to_s)
+						@params[e[1]]=param_value unless param_value.to_s.empty?
+					end
+					@params[:page] = Param::FIRST_PAGE
+					@params[:platform] = Param::WEB_URL
+					@params[:out_offset_type] = Param::OUT_GAODE_OFFSET
+					@params[:offset_type] = Param::GAODE_OFFSET
+					@params[:format] = Param::JSON
+					@params[:limit] = Param::DEFAULT_LIMIT
+					@params[:sort] = Param::SORT_BY_STAR
+					true
+				end
+				def find_businesses_by_region
+					
+				end
+				def get_single_business(business_id)
+					@uri = URI File.join(BaseUri,__method__.to_s)
+					@params['business_id'] = business_id
+				end
+				def set_category(category)
+					@params[:category] = category
+				end
+			end
+		end
+	end
+end

data/lib/espider/api/dianping/metadata.rb

@@ -0,0 +1,28 @@
+require 'active_support/core_ext/string/inflections'
+module ESpider
+	module API
+		module Dianping
+			class Metadata < ESpider::API::Dianping::Base
+				BaseUri = File.join(Domain,Version,self.name.demodulize.downcase)
+				Methods = [
+					'get_cities_with_businesses',
+					'get_regions_with_businesses',
+					'get_categories_with_businesses',
+					'get_cities_with_deals',
+					'get_regions_with_deals',
+					'get_categories_with_deals',
+					'get_cities_with_coupons',
+					'get_regions_with_coupons',
+					'get_categories_with_coupons'
+				]
+				Methods.each do |method|
+					define_method(method){init_methods(method)}
+				end
+				def init_methods(method_name)
+					@uri = URI File.join(BaseUri,method_name.to_s)
+					calc_digest
+				end
+			end
+		end
+	end
+end

data/lib/espider/api/dianping/params.rb

@@ -0,0 +1,34 @@
+module ESpider
+	module API
+		module Dianping
+			module Param
+				EMPTY = ""
+				NO_OFFSET = 0
+				GAODE_OFFSET = 1
+				TUBA_OFFSET = 2
+				OUT_GAODE_OFFSET = 1
+				OUT_TUBA_OFFSET = 2
+				DEFAULT_RADIUS = 1000
+				WEB_URL = 1
+				MOBILE_URL = 2
+				HAS_COUPON = 1
+				NO_COUPON = 0
+				HAS_DEAL = 1
+				NO_DEAL = 0
+				SORT_DEFAULT = 1
+				SORT_BY_STAR = 2
+				SORT_BY_RATE = 3
+				SORT_BY_ENV = 4
+				SORT_BY_SERV = 5
+				SORT_BY_COMM = 6
+				SORT_BY_DIST = 7
+				SORT_BY_MPRICE_LOW = 8
+				SORT_BY_MPRICE_HIGH = 9
+				JSON = 'json'
+				XML = 'xml'
+				DEFAULT_LIMIT = 20
+				FIRST_PAGE = 1
+			end
+		end
+	end
+end

data/lib/espider/exceptions.rb

@@ -0,0 +1,2 @@
+class QunarIpBlockException < RuntimeError ; end
+class QunarMixKeyParsedException < RuntimeError ; end

data/lib/espider/front/baidu/map/hotel.rb

@@ -0,0 +1,12 @@
+require 'espider/front/baidu/map/hotel/list'
+require 'espider/front/baidu/map/hotel/detail'
+module ESpider
+	module Front
+		module Baidu
+			module Map
+				module Hotel
+				end
+			end
+		end
+	end
+end

data/lib/espider/front/baidu/map/hotel/detail.rb

@@ -0,0 +1,60 @@
+require 'nokogiri'
+require 'httparty'
+require 'uri'
+module ESpider
+	module Front
+		module Baidu
+			module Map
+				module Hotel
+					class Detail
+						HotelUri = "http://map.baidu.com/detail"
+						def initialize(uid)
+							url = HotelUri + "?qt=ninf&uid=#{uid}"
+							@hotel = JSON.parse HTTParty.get url
+							true
+						end
+						def hotel_name
+							@hotel['content']['name']
+						end
+						def hotel_addr
+							@hotel['content']['addr']
+						end
+						def hotel_tel
+							@hotel['content']['phone']
+						end
+						def hotel_geo
+							@hotel['content']['geo'].split(/;/)[0].sub(/\d+\|/,"").split(',')
+						end
+						def hotel_star
+							@hotel['content']['ext']['rich_info']['level']
+						end
+						def hotel_category
+							@hotel['content']['ext']['rich_info']['category']
+						end
+						def hotel_price
+							@hotel['content']['ext']['detail_info']['price']
+						end
+						def hotel_facility
+							@hotel['content']['ext']['rich_info']['inner_facility']
+						end
+						def hotel_short_comm
+							@hotel['content']['ext']['detail_info']['short_comm']
+						end
+						def hotel_review
+							reviews = []
+							@hotel['content']['ext']['review'].each do |r|
+								reviews << r if r['name']!='elong'
+							end
+							reviews
+						end
+						def hotel_image(uid)
+							url = HotelUri + "?qt=img&uid=#{uid}"
+							parsed_json = JSON.parse HTTParty.get url
+							parsed_json['images'].map{ |i| i['imgUrl']}
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/lib/espider/front/baidu/map/hotel/list.rb

@@ -0,0 +1,46 @@
+require 'nokogiri'
+require 'httparty'
+require 'uri'
+module ESpider
+	module Front
+		module Baidu
+			module Map
+				module Hotel
+					class List
+						attr_reader :total
+						ListUri = "http://map.baidu.com/"
+						def initialize(city_cn)
+							@wd = URI.encode city_cn + '+' + '酒店'
+							@pn=1
+							url = ListUri+"?newmap=1&reqflag=pcmap&biz=1&qt=s&wd=#{@wd}&c=1&src=0&wd2=&sug=0&l=4&from=webmap&tn=B_NORMAL_MAP&nn=0&ie=utf-8"
+							@hotels = HTTParty.get url
+							@total = count_page_num
+						end
+						def next_page
+							return false if (@pn+1)*10 > @total
+							@pn += 1
+							url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&qt=s&wd=#{@wd}&c=1&src=0&sug=0&l=4&from=webmap&rn=10&pl_data_type=hotel&pn=#{@pn}"
+							p url
+							@hotels = HTTParty.get url
+							true
+						end
+						def list
+							parsed_json = JSON.parse @hotels
+							results = []
+							return nil if parsed_json['content'].nil?
+							parsed_json['content'].each do |hotel|
+								results << hotel
+							end
+							results
+						end
+						private
+						def count_page_num
+							parsed_json = JSON.parse @hotels
+							parsed_json['result']['total']
+						end
+					end
+				end
+			end
+		end
+	end
+end