qunar 0.0.32 → 0.1.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89b8b2cf234cc5fedd22d76f0ded9fafd1e77cef
-  data.tar.gz: d6513eb1642959ce237917743723bdcc09e6da87
+  metadata.gz: 851aeecc15e69334b7f294ee335552a76c0527b1
+  data.tar.gz: abac57dec98f03e9454b267ebda9f759b2ba76c3
 SHA512:
-  metadata.gz: 8c6c84123c0e521edbe618f8906647f868221ec1e631222809dbdee38419be990161111d7022199b2c1970c756dd416ec9c76e8d57353d81590a647f93a54e13
-  data.tar.gz: bbaaad66c2e3dfc81ee423864d903ed104360f3463d9c8c6915234d8d3953678c88e08b7a99d50b5b35a619d4e23fd8ac9dfd1ce0cc820072adaa52e783b43a7
+  metadata.gz: 310aab733356aaa23ffc2b420ba70f06e4ef9efb1a6f497d0c992f178e2d23e72c3e9b94abd10c814594d8f8c01ee8f4201bbd7e28716afe423018e63869cb80
+  data.tar.gz: fbb907544e968ee7e06080ee2ad7d3e17c7b9be9723740b10c49b6ee1fefd0e378f761bc50c5cc00604a248beacc8e8fbf8b466e8f4858125317e7382e808280

data/Gemfile

@@ -1,4 +1,7 @@
-source 'https://rubygems.org'
+source 'https://ruby.taobao.org'
 
 # Specify your gem's dependencies in qunar.gemspec
-gemspec
+gem 'httparty','=0.13.3'
+gem 'nokogiri','=1.6.5'
+gem 'capybara','=2.4.4'
+gem 'capybara-webkit','=1.3.1'

data/lib/qunar/hotel.rb

@@ -1,5 +1,3 @@
-require 'nokogiri'
-require 'httparty'
 module Qunar
   class Hotel
     BASEURI = "http://hotel.qunar.com"

data/lib/qunar/hotel_list.rb

@@ -0,0 +1,109 @@
+module Qunar
+  class HotelList
+
+    def initialize(city_url)
+      @city = city_url
+    end
+
+    # Call ajax for more hotels of each page
+    def hotels_in_ajax(page_id)
+      sleep(2)   # sleep for 2 seconds
+      @url = "http://touch.qunar.com//h5/hotel/hotellist?cityUrl=#{@city}&page=#{page_id}&tpl=hotel.hotelListTpl"
+      page = HTTParty.get(@url.to_s)
+      @html = Nokogiri::HTML page
+      pois = @html.xpath("//li[@class='qn_bt']")
+      if !pois.empty?
+        hotels = pois.collect { |hotel|
+          title = hotel.at("./div[@class='info qn_arrow_grey r']/div[@class='title']")
+          item = { :city=>@city, :hotel_id=>hotel['data-seq'], :hotel_name=>title.nil? ? nil : title.text.strip }        
+        }
+      end
+      hotels
+    end 
+
+=begin
+   include Capybara::DSL
+
+
+    def initialize(city_en)
+      raise ArgumentError, "initizlize: Argument should be string " if !city_en.instance_of?(String)
+      #Global Setting
+      Capybara.run_server = false
+      Capybara.current_driver = :webkit   #selenium/poltergeist
+
+      # Use XPath as the default selector for the find method
+      Capybara.default_selector = :xpath
+
+      Capybara.app_host = 'http://hotel.qunar.com'
+      # Set user agent 
+       page.driver.header 'User-Agent', "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
+
+      # For poltergeist
+      #page.driver.headers={'User-Agent'=> "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"}
+
+      @city = city_en
+      # log msg
+      puts "正在访问: http://hotel.qunar.com/city/#{@city}"
+      page.visit "/city/#{@city}"
+      @page = Nokogiri::HTML page.html
+    end
+
+    # Wait for ajax, a selector must be passed as capybara will wait for the element
+    def wait_for_ajax(selector)
+      puts "\e[32m Waiting for ajax...\e[0m"
+      # Retry 3 times at most, if still not found, give up
+      limiter = 0
+      begin
+        page.find(selector)  
+      rescue Capybara::ElementNotFound
+        limiter+=1
+        retry if limiter<3
+      end
+    end
+    
+    def hotels_in_page
+      # Scroll down and trigger ajax
+      page.execute_script "window.scrollTo(0, document.body.scrollHeight)"
+      self.wait_for_ajax("//span[@class='hotel_num js_hotel_num'][text()='30']")
+      hotels = []
+      @page = Nokogiri::HTML page.driver.browser.body 
+      @page.css("div.b_result_box").each { |hotel|
+        item = {:city => @city, :hotel_id => hotel['data-id']}
+        item[:hotel_name] = hotel.search("a[@class='e_title js_list_name']").first.text
+        hotels << item
+      }
+      hotels
+    end
+
+    # click `下一页` to navigate to next page
+    def click_next?
+      next_button = page.all("//li[@class='item next ']/a").first
+      return false if next_button.nil?
+      puts "Next page"
+      next_button.click
+      self.wait_for_ajax("//span[@class='hotel_num js_hotel_num'][text()='15']")
+      return true
+    end
+
+    # Get the page number
+    def page_count
+      a_node = @page.css("li.pager_count")
+      a_node.empty? ? 0 : a_node.first.text.split('/')[1].to_i 
+    end
+
+    # Get the screen_shot
+    def screen_shot
+      page.save_screenshot("#{Time.now}.png")
+    end
+=end   
+
+    # Html file
+    def html
+      @html
+    end
+
+  end
+end
+
+
+      

data/lib/qunar/rank.rb

@@ -1,68 +1,66 @@
-require 'capybara'
-require 'nokogiri'
-require 'capybara/poltergeist'
-#require 'capybara-webkit'
-#require 'selenium-webdriver'
-
 module Qunar
   class Rank
     include Capybara::DSL
-
+  
     def initialize(hotelId)
-
       raise "id should be string" if !hotelId.instance_of?(String)
-      Capybara.current_driver = :poltergeist   #selenium / webkit
+      Capybara.current_driver = :webkit   # :selenium/:poltergeist
       Capybara.default_selector = :xpath
       Capybara.app_host = 'http://hotel.qunar.com'
-      page.driver.headers = { "User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.77.4 (KHTML, like Gecko) Version/7.0.5 Safari/537.77.4" }
-      @HotelSEQ = hotelId                #hotelId is "cityurl_hotelcode"
-      @cityurl = hotelId[0..hotelId.rindex("_",-1)-1]
-      @hotelcode = hotelId[hotelId.rindex("_",-1)+1..hotelId.length]
+
+      # Header for webkit
+      page.driver.header 'User-Agent', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.77.4 (KHTML, like Gecko) Version/7.0.5 Safari/537.77.4"
+
+      @HotelSEQ = hotelId   #hotelId is "cityurl_hotelcode"
+      @cityurl, suffix, @hotelcode = hotelId.rpartition('_')
       @uri = URI(File.join('/city', @cityurl, "dt-#{@hotelcode}"))
-      visit @uri
-      traffic_size = 0
-      maximum = 0
-      loop do 
-        sleep 3
-        traffic_size = page.driver.network_traffic.size
-        puts "Get dynamic request #{traffic_size}"
-        break if traffic_size == maximum 
-        maximum = traffic_size if maximum < traffic_size
-      end
-      # k = 0
-      all('//td[@class="book-ct"]//*[text()="展开报价"] | //td[@class="book-ct"]//*[text()="全部订完"]').each do |spage|
-        puts 'cp 2-1'
-        spage.trigger('click')
-        puts 'cp 2-2'
+      page.visit @uri
+
+      limiter = 0
+      begin
+        find("//td[@class='book-ct']//*[text()='展开报价']")
+        page.all('//td[@class="book-ct"]//*[text()="展开报价"] | //td[@class="book-ct"]//*[text()="全部订完"]').each do |spage|
+          puts 'cp 2-1'
+          spage.click
+          puts 'cp 2-2'
+        end
+        puts 'cp 3'
+        # Expand hidden otas
+        page.all("//div[@class='more-quote js-room-more']/a").each do |link|
+          link.click
+        end
+      rescue =>e
+        limiter+=1
+        retry if limiter<3
+        raise e
       end
-      puts 'cp 3'
+
       #@page = Capybara.current_session.driver.browser.page_source    #use when choose capybara driver selenium
-      @page = Capybara.current_session.driver.browser.body    #use when choose capybara driver webkit
-      @hotel = Nokogiri::HTML @page
-      
+      @hotel = Nokogiri::HTML page.html  #use when choose capybara driver webkit
     end
 
+    # Get the rooms details
     def rooms
       rooms = []
     
       # room
-      @hotel.search("//li[@class='default-type defaultpricetype similar-expand']").each do |li|
+      @hotel.search("//li[@class='default-type defaultpricetype similar-expand' or @class='default-type defaultpricetype similar-expand b_result_item_on']").each do |li|
         #puts 'one roomtype found'
         room = {}
         type = li.search("span[@class='type-name js-p-name']").first.text.strip
-        room['room_type'] = type
-        room['otas'] = []
+        room[:room_type] = type
+        room[:otas] = []
         # ota
         li.search("div[@class='similar-type-agent-list']/div/div").each do |row|
           otas = {}
 
           ota = row.search("div[@class='profile-tit']").first
           ota = ota.text.strip
-          otas['ota'] = ota
+          otas[:ota] = ota
 
           bookable = row.search("a[@class='btn-book']/span")
           unless bookable.empty?
-            bookable == bookable.first.text.strip
+            bookable = bookable.first.text.strip
             if bookable == '订完'
               bookable = false
             else
@@ -71,15 +69,23 @@ module Qunar
           else
             bookable = true
           end
-          otas['bookable'] = bookable
+          otas[:bookable] = bookable
 
-          insurance = row.search("p[@class='dbt-ct']").first
+          insurance = row.search("a[@class='js-guarantee-plan']").first
           if insurance == nil
             insurance = false
           else
             insurance = true
           end
-          otas['insurance'] = insurance
+          otas[:safe] = insurance
+
+          guarantee = row.search("cite[text()='需担保']").first
+          if guarantee == nil
+            guarantee = false
+          else
+            guarantee = true
+          end
+          otas[:guarantee] = guarantee
 
           coupon = row.search("span[@class='fan']/em[@class='pr']").first
           if coupon == nil
@@ -87,18 +93,15 @@ module Qunar
           else
             coupon = coupon.text.sub(/¥/,'').strip.to_i
           end
-          otas['coupon'] = coupon
-
+          otas[:coupon] = coupon
+          
           reduce = row.search("span[@class='fan lapse']/em[@class='pr']").first
           if reduce == nil
             reduce = 0
           else
             reduce = reduce.text.sub(/¥/,'').strip.to_i
           end
-          otas['reduce'] = reduce
-
-          reserve_detail = row.search("div[@class='clrfix order-detail']").first.text.strip
-          otas['reserve_detail'] = reserve_detail
+          otas[:reduce] = reduce
 
           prepay = row.search("span[@title='需要预先支付房款']").first
           if prepay == nil
@@ -106,17 +109,16 @@ module Qunar
           else
             prepay = true
           end
-          otas['prepay'] = prepay
-
+          otas[:prepay] = prepay
 
 
           price = row.search("p[@class='final-price ']/b").first.text.strip
           neat_price = row.search("p[@class='count_pr']").first.children[1].to_s
 
-          otas['price'] = price.to_i
-          otas['neat_price'] = neat_price.to_i
+          otas[:original_price] = price.to_i
+          otas[:final_price] = neat_price.to_i
 
-          room['otas'] << otas
+          room[:otas] << otas
         end
         rooms << room
         
@@ -124,5 +126,65 @@ module Qunar
       return rooms
     end
 
+   # Get the Chinese name of the hotel
+    def name         
+      name  = @hotel.search("//div[@class='htl-info fr']//span")
+      if name.empty?
+        name = '该酒店不存在'
+      else
+        name = name.first.text.strip
+      end
+      return name
+    end
+
+   # Get the level of the hotel
+    def star         
+      name_tag = @hotel.search('//div[@class="htl-info fr"]//em[@class="dangci"]').first
+      return 0 if name_tag.nil?
+      if name_tag.text.include?'经济型'
+        1
+      elsif name_tag.text.include?'舒适型'
+        3
+      elsif name_tag.text.include?'高档型'
+        4
+      elsif name_tag.text.include?'豪华型'
+        5
+      elsif name_tag.text.include?'star20'
+        2
+      elsif name_tag.text.include?'star50'
+        5
+      else
+        0
+      end
+    end
+
+    # Get the Chinese name of the City
+    def citynameCn
+      page.evaluate_script("(function f() { return cityName; }());")
+    end
+
+    # Get the English name of the City
+    def cityNameEn
+      page.evaluate_script("(function f() { return cityNameEn; }());")
+    end
+
+    # Get the Hotel details
+    def hotelInfo
+
+      {  :hotel_id => @HotelSEQ, :hotel_name => self.name,
+         :city => self.cityNameEn, :city_cn => self.citynameCn, 
+         :star => self.star  }
+    end
+
+    # Get the screen_shot
+    def screen_shot
+      page.save_screenshot("#{Time.now}.png")
+    end
+
+    # Html file
+    def html
+      @hotel
+    end
+
   end
 end

data/lib/qunar/version.rb

@@ -1,3 +1,3 @@
 module Qunar
-  VERSION = "0.0.32"
+  VERSION = "0.1.37"
 end

data/lib/qunar.rb

@@ -1,7 +1,23 @@
+require 'capybara'
+require 'nokogiri'
+require 'open-uri'
+require 'httparty'
+require 'capybara-webkit'
 require 'qunar/version'
 require 'qunar/hotel'
 require 'qunar/rank'
+require 'qunar/hotel_list'
 
 module Qunar
-  # Your code goes here...
+ # City list 
+ def self.city
+    pages = Nokogiri::HTML(open(URI('http://hotel.qunar.com/sitemap/')))
+    @city_list = []
+    pages.xpath("//li/a").each do |li|
+      item = {:city_Cn=>li.text, :city_En=>li['href'][/\w+/]}
+      @city_list << item
+    end
+    @city_list
+  end
+
 end

data/qunar.gemspec

@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.1.0"
 end

data/spec/hotel_list_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+  describe Qunar::HotelList do
+    sample = Qunar::HotelList.new('beijing_city')
+
+    it "has 15 hotels in one page" do
+      expect(sample.hotels_in_ajax(1).size).to eq 15
+    end
+end

data/spec/hotel_spec.rb

@@ -1,4 +1,4 @@
-require 'qunar/hotel'
+require 'spec_helper'
 
 describe Qunar::Hotel do 
 	bj_sample = Qunar::Hotel.new('beijing_city_2933')

data/spec/rank_spec.rb

@@ -1,10 +1,8 @@
-require 'qunar/rank'
+require 'spec_helper'
 
 describe Qunar::Rank do
   sample = Qunar::Rank.new('beijing_city_2933')
-
-  sz_sample = Qunar::Rank.new('shenzhen_6993')
-
+  p sample.rooms
   it "has 7 rooms" do
     # when directly check the hash object the Chinese charactervalue is saved as format like "\u6807\u51C6\u53CC\u5E8A\u623F", it is not been parsed here, so it is not equal to chinese charactor, it's just a sequnce of coding,it needs to be parse,then it could show as Chinese character.
     # room = {"room_type" => "\u6807\u51C6\u53CC\u5E8A\u623F", "ota" => "\u827A\u9F99\u65C5\u884C\u7F51", "bookable" => "\u9884\u8BA2", "insurance" => true, "coupon" => "0", "prepay" => false, "price" => "918", "neat_price" => "918"}
@@ -12,88 +10,88 @@ describe Qunar::Rank do
   end
 
   it "first room names '标准双床房'" do 
-    expect(sample.rooms.first['room_type']).to eq '标准双床房'
+    expect(sample.rooms.first[:room_type]).to eq '标准双床房'
   end
 
   it "second room names '标准大床房'" do 
-    expect(sample.rooms[1]['room_type']).to eq '标准大床房'
+    expect(sample.rooms[1][:room_type]).to eq '标准大床房'
   end
 
 
 
   it "has 20 otas for the first room type '标准双床房'" do 
-    expect(sample.rooms.first['otas'].size).to eq 20
+    expect(sample.rooms.first[:otas].size).to eq 20
   end
 
   it "has 20 otas for the second room type '标准大床房'" do 
-    expect(sample.rooms[1]['otas'].size).to eq 20
+    expect(sample.rooms[1][:otas].size).to eq 20
   end 
 
 
   it "first ota of first roomtype names '艺龙旅行网'" do
     #puts subject.rooms.first['otas'].first['ota']
-    expect(sample.rooms.first['otas'].first['ota']).to eq '艺龙旅行网'
+    expect(sample.rooms.first[:otas].first[:ota]).to eq '艺龙旅行网'
   end
 
 
   it "first ota of second roomtype names Expedia" do
     #puts subject.rooms.first['otas'].first['ota']
-    expect(sample.rooms[1]['otas'].first['ota']).to eq 'Booking.com'
+    expect(sample.rooms[1][:otas].first[:ota]).to eq 'Booking.com'
   end
 
   it "first ota of first roomtype is bookable" do
-    expect(sample.rooms.first['otas'].first['bookable']).to eq true
+    expect(sample.rooms.first[:otas].first[:bookable]).to eq true
   end
 
   it "last ota of first roomtype is not bookable" do
-    expect(sample.rooms.first['otas'][-1]['bookable']).to eq false
+    expect(sample.rooms.first[:otas][-1][:bookable]).to eq false
   end
 
 
   it "first ota of first roomtype need insurance" do
-    expect(sample.rooms.first['otas'].first['insurance']).to eq true
+    expect(sample.rooms.first[:otas].first[:safe]).to eq true
   end
 
   it "second ota of first roomtype not need insurance" do
-    expect(sample.rooms.first['otas'][1]['insurance']).to eq false
+    expect(sample.rooms.first[:otas][1][:safe]).to eq false
   end
 
   it "first ota of first roomtype coupon is 0" do
-    expect(sample.rooms.first['otas'].first['coupon']).to eq 0
+    expect(sample.rooms.first[:otas].first[:coupon]).to eq 0
   end
 
   it "first ota of first roomtype in shenzhen coupon is 40" do
-    expect(sz_sample.rooms.first['otas'].first['coupon']).to eq 40
+    expect(sz_sample.rooms.first[:otas].first[:coupon]).to eq 40
   end
 
   it "first ota of first roomtype reserve_detail is '标准双床房 - 仅提供住宿 - 免费取消'" do
-    expect(sample.rooms.first['otas'].first['reserve_detail']).to eq '标准双床房 - 仅提供住宿 - 免费取消'
+    expect(sample.rooms.first[:otas].first[:reserve_detail]).to eq '标准双床房 - 仅提供住宿 - 免费取消'
   end
 
 
   it "first ota of first roomtype need prepay" do
-    expect(sample.rooms.first['otas'][2]['prepay']).to eq true
+    expect(sample.rooms.first[:otas][2][:prepay]).to eq true
   end
 
   it "second ota of first roomtype not need prepay" do
-    expect(sample.rooms.first['otas'][1]['prepay']).to eq false
+    expect(sample.rooms.first[:otas][1][:prepay]).to eq false
   end
 
   it "first ota of first roomtype price" do
-    expect(sample.rooms.first['otas'].first['price']).to eq 918
+    expect(sample.rooms.first[:otas].first[:original_price]).to eq 918
   end
 
   it "fifth ota of first roomtype price" do
-    expect(sample.rooms.first['otas'][4]['price']).to eq 638
+    expect(sample.rooms.first[:otas][4][:original_price]).to eq 638
   end
 
   it "first ota of first roomtype neat_price" do
-    expect(sample.rooms.first['otas'].first['neat_price']).to eq 918
+    expect(sample.rooms.first[:otas].first[:final_price]).to eq 918
   end
 
   it "sixth ota of first roomtype neat_price" do
     #puts subject.rooms.first['otas'][5]['neat_price']
-    expect(sample.rooms.first['otas'][4]['neat_price']).to eq 734
+    expect(sample.rooms.first[:otas][4][:final_price]).to eq 734
   end
   
 end

data/spec/spec_helper.rb

@@ -78,7 +78,7 @@ RSpec.configure do |config|
 end
 
 
-
+require 'qunar'
 
 $qunar_hotel= File.read(File.join(__dir__, 'samples', 'qunar_hotel.html'))
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qunar
 version: !ruby/object:Gem::Version
-  version: 0.0.32
+  version: 0.1.37
 platform: ruby
 authors:
 - JusticeChow
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-26 00:00:00.000000000 Z
+date: 2015-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 description: "包含去哪酒店基本信息以及各ota报价信息"
 email:
 - zhengyi.zhou@corp.elong.com
@@ -53,12 +67,13 @@ files:
 - Rakefile
 - lib/qunar.rb
 - lib/qunar/hotel.rb
+- lib/qunar/hotel_list.rb
 - lib/qunar/rank.rb
 - lib/qunar/version.rb
 - qunar.gemspec
+- spec/hotel_list_spec.rb
 - spec/hotel_spec.rb
 - spec/rank_spec.rb
-- spec/samples/qunar_hotel.html
 - spec/spec_helper.rb
 homepage: ''
 licenses:
@@ -80,12 +95,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: "去哪酒店数据明细"
 test_files:
+- spec/hotel_list_spec.rb
 - spec/hotel_spec.rb
 - spec/rank_spec.rb
-- spec/samples/qunar_hotel.html
 - spec/spec_helper.rb