daodao 0.0.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac9c046fcc208a74bb7645267d5def002c35f12c
-  data.tar.gz: 0d9bbd10a32402f9be3f8af7b971e9ac070b1216
+  metadata.gz: 94e13e69f1a94e9ef2457e87b79d059580e4e62b
+  data.tar.gz: b6ff9806987c8b4b28ac3fdbb8802c4728676165
 SHA512:
-  metadata.gz: 1070827a478997f46f8a44018fc3d4239a7777b6ffdf9d810157c2366279b271536187b1032545f496f775251de8ea1c75fc4c08feaedf6e2c5b37b381945b83
-  data.tar.gz: 440fcc776db47417224b6e3a4d4dbdec9d9cbd0a2bff8a0f5ce0e28bace35be89880d5ffd2a36f064fd99b2165f08e5767cacd4ee18b4d276e25eac19c2e221d
+  metadata.gz: 76c7b442362b441f4a038bccf9bb74a4eadcc802692c37049200f2e870f187caffafed1afea874bf05b4228976cc9da6923f1e277da39b8e10f1c495fd498a28
+  data.tar.gz: 1e302326ab4b61b556dad26e114055f29e3d3f067bfc7dba09a8adb6e6cdb9eb11e4e81149058587a2f795dec4d44451323fbd03266ef12ab1d44cba50e65dc4

data/.gitignore

@@ -12,3 +12,4 @@
 *.o
 *.a
 mkmf.log
+*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in daodao.gemspec
-gemspec
+source 'https://ruby.taobao.org'
+gem 'nokogiri','=1.6.6.2'
+gem 'capybara', '=2.4.4'
+gem 'capybara-webkit','=1.3.1'

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2014 charles
+Copyright (c) 2014 timlentse
 
 MIT License
 

data/README.md

@@ -1,8 +1,8 @@
-# Daodao
+### Daodao
 
-TODO: Write a gem description
+Daodao is a Ruby gem that crawls the ota's ranking and the rooms details in [www.daodao.com](http://www.daodao.com)
 
-## Installation
+### Installation
 
 Add this line to your application's Gemfile:
 
@@ -17,12 +17,41 @@ And then execute:
 Or install it yourself as:
 
     $ gem install daodao
+  
+#### Getting started
 
-## Usage
+```sh
+ $ touch test.rb
+ $ vi test.rb
+```
+ Add some codes like the following   
+  
+```ruby    
+require 'daodao' 
+  uri = "/Hotel_Review-g298555-d2682464-Reviews-Four_Seasons_Hotel_Guangzhou-Guangzhou_Guangdong.html"
+  sample = Daodao::Rank.new(uri)
+  puts 'The rooms details are as follow :'
+  puts sample.rooms_details
+```
+This will crawl the rooms details of the target hotel for you.   
+
+`Note:` You can pass other uri to `initialize` representing the hotel page. 
+
+#### Get ota's ranking
+
+```ruby   
+require 'daodao'
+
+  ranking = sample.rank
+  puts "The ota's ranking are as follow :"
+  puts ranking
+```   
+
+#### Bug
+1. The speed is still a problem   
 
-TODO: Write usage instructions here
 
-## Contributing
+### Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/daodao/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)

data/Rakefile

@@ -1,2 +1,2 @@
 require "bundler/gem_tasks"
-
+  

data/daodao.gemspec

@@ -6,10 +6,10 @@ require 'daodao/version'
 Gem::Specification.new do |spec|
   spec.name          = "daodao"
   spec.version       = Daodao::VERSION
-  spec.authors       = ["charles"]
-  spec.email         = ["charles.liu@corp.elong.com"]
-  spec.summary       = %q{daodao hotel list}
-  spec.description   = %q{daodao hotel list}
+  spec.authors       = ["timlen"]
+  spec.email         = ["tinglenxan@gmail.com"]
+  spec.summary       = %q{}
+  spec.description   = %q{daodao hotel list.}
   spec.homepage      = ""
   spec.license       = "MIT"
 
@@ -20,4 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.1.0"
+
 end

data/lib/daodao.rb

@@ -1,5 +1,23 @@
-require "daodao/version"
-
+require 'nokogiri'
+require 'capybara'
+require 'capybara-webkit'
+require 'open-uri'
+require 'daodao/version'
+require 'daodao/rank'
+require 'daodao/hotel'
+require 'daodao/hotel_list'
 module Daodao
-  # Your code goes here...
+  def self.rank(url)
+    page = Nokogiri::HTML(open(url))
+    row = {}
+    row[:hotel_id] = url.split('-')[2]
+    row[:hotel_name] = page.xpath('//span[@itemprop="name"]').text
+    rank=1
+    page.css("div.provider_meta_overview").each { |item|
+      key = 'rank'+rank.to_s
+      row[key.to_sym] = item.css("span.providerLogo_96x41").xpath('./img/@alt').first.value
+      rank+=1
+    }
+    row
+  end
 end

data/lib/daodao/hotel.rb

@@ -0,0 +1,49 @@
+module Daodao
+  class Hotel
+
+     # Crawl page from the desired hotel page
+     # @parmas: hotel_id, string 
+    def initialize(hotel_id)
+      raise "The argument of class HotelInfo should be string" if !hotel_id.instance_of?(String)
+      @id = hotel_id
+      @url = URI("http://daodao.com/Hotel_Review-#{hotel_id}")
+      @pages = Nokogiri::HTML(open(@url))
+    end
+      
+    def name
+      @pages.xpath('//span[@itemprop="name"]').text
+    end
+    
+    def id
+      @id
+    end
+
+    def stars
+      str = @pages.xpath('//div[@class="stars"]/span/img/@alt').first.value
+      str[/\d+/].to_i
+    end
+    
+    def locality
+      @pages.at_css('span.locality').text
+    end
+
+    def country
+      @pages.at_css('span.country-name').text
+    end
+  
+    def address
+      #TODO
+      #@pages.css('span.street-address')
+    end
+    
+    def rank
+      text = @pages.xpath('//div[@class="slim_ranking"]/b[@class="rank_text"]').text
+      text[/\d+/].to_i
+    end
+
+    def desc
+       @pages.xpath('//span[@class="desc"]/text()')[0].text
+    end
+
+  end
+end

data/lib/daodao/hotel_list.rb

@@ -0,0 +1,33 @@
+module Daodao
+  class HotelList
+  START_URL = "http://www.daodao.com/dpages/sitemap/hotels.html"
+    def initialize
+      @page = Nokogiri::HTML(open(URI(START_URL)))
+      @page_amount = @page.xpath("//div[@id='page_list']/p/a").size 
+    end
+      
+  # Fetch the hotels in each page
+    def hotels_in_page(page_id)
+      hotel_list = []
+        if page_id > 1      
+          url = "http://www.daodao.com/dpages/sitemap/hotels-#{page_id}.html"
+        else 
+          url = "http://www.daodao.com/dpages/sitemap/hotels.html"
+        end
+        page = Nokogiri::HTML(open(URI(url)))
+        page.xpath("//dl/a").each { |hotel|
+          item = {}
+          item[:name] = hotel.text
+          item[:url] = hotel['href']
+          hotel_list.push(item)
+        }
+      return hotel_list
+    end
+    
+  # Get the page's number
+    def page_amount
+      @page_amount
+    end
+
+  end
+end

data/lib/daodao/rank.rb

@@ -0,0 +1,92 @@
+module Daodao
+  class Rank
+    include Capybara::DSL
+
+    def initialize(uri)
+      Capybara.current_driver = :webkit 
+      
+      Capybara.default_selector = :xpath
+
+      Capybara.app_host ="http://daodao.com"
+      page.driver.header "User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.77.4 (KHTML, like Gecko) Version/7.0.5 Safari/537.77.4" 
+      puts " \e[34mBuilding Session...\e[0m"
+      @uri = uri
+      page.visit(URI(uri))
+      # Click the div to navigate hidden info
+      page.all("//div[@class='tab ']").each do |node|
+        node.trigger('click')
+      end
+      # Expand all divs tagged with '展开报价'
+      page.all("div[@class='provider_meta_overview  mskQC_Meta_RoomTab msk']//*[text()='展开报价']").each do |layout|
+        puts '展开报价...'
+        layout.trigger('click')
+      end
+      @page = Nokogiri::HTML(page.html)
+    end
+
+    # Get all the rooms details 
+    # @Return: an array, a list of rooms details
+    def rooms_details
+      otas = []
+      @page.css("div.provider_meta_overview").each { |item|
+        row = {}
+        row[:ota_name] = item.css("span.providerLogo_96x41").xpath('./img/@alt').first.value
+
+        state = item.css("span.unavailable")
+        if state.empty?
+          row[:ota_state] = "有房"
+          row[:lowest_price] = item.css("span.price").text.sub(/¥/,'').strip.to_i
+        else
+          row[:ota_state] = "也许有房"
+          row[:lowest_price] = nil
+        end
+
+        rooms = []
+        dropdown_room_list = @page.css("div.provider_rooms_details")[otas.size]
+        dropdown_room_list.xpath("./a").each do |room_details|       
+          room = {}
+          room[:room_type] = room_details.at_css("span.room_type").text
+
+          price_tax_fees = room_details.at_css("span.price_tax_fees_wrap")
+            room[:room_price] = price_tax_fees['price'].to_i
+            room[:room_tax] = price_tax_fees['taxfee'].to_i
+
+          return_amount_node = room_details.css("span.rebate_amount")
+            room[:return_amount] = return_amount_node.empty? ?  0 : return_amount_node.text.sub(/¥/,'').to_i
+
+          room[:room_state] = room_details.at_css("span.available").text
+          rooms << room
+        end
+        row[:rooms_details] = rooms
+        otas << row
+      }
+      return otas
+    end
+
+    def name
+      @page.xpath('//span[@itemprop="name"]').text
+    end
+    
+    def id
+      @uri.split('-')[2]
+    end
+
+    def rank
+      row = {}     
+      row[:hotel_id] = self.id
+      row[:hotel_name] = self.name
+      rank=1
+      @page.css("div.provider_meta_overview").each { |item|
+        key = 'rank'+rank.to_s
+        row[key.to_sym] = item.css("span.providerLogo_96x41").xpath('./img/@alt').first.value
+        rank+=1
+      }
+      return row
+    end
+       
+    def screen_shot
+      page.save_screenshot("#{Time.now}.png")
+    end
+
+  end
+end

data/lib/daodao/version.rb

@@ -1,3 +1,3 @@
 module Daodao
-  VERSION = "0.0.1"
+  VERSION = "0.1.3"
 end

data/spec/hotel_list_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+  describe Daodao::HotelList do
+    object = Daodao::HotelList.new
+
+    it "The page amount is '307' " do
+      expect(object.page_amount).to eq 307
+    end
+    
+    it " The first hotel's name is '168大酒店' " do
+      expect(object.hotels_in_page(1)[0][:name]).to eq "168大酒店"
+    end
+
+end

data/spec/hotel_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe Daodao::Hotel do 
+  # hotel in guangzhou
+	gz_sample = Daodao::Hotel.new("d2682464")
+
+	it "has name '四季酒店'" do 
+		expect(gz_sample.name).to eq '四季酒店'
+	end
+
+   it "has id 'd2682464'" do
+    expect(gz_sample.id).to eq 'd2682464'
+  end
+
+	it "has stars 5.0星级" do
+		expect(gz_sample.stars).to eq 5
+	end
+
+	it "has locality '广州市'" do
+		expect(gz_sample.locality).to eq "广州市"
+	end
+
+	it "has country name '广东省' " do
+		expect(gz_sample.country).to eq "广东省"
+	end
+
+	it "has rank '3'" do
+		expect(gz_sample.rank).to eq 3
+	end
+	
+	it "has desc '\n' " do
+		expect(gz_sample.desc).to eq "\n"
+  end
+
+	#=====================================================================================================
+  # hotel in beijing
+	bj_sample = Daodao::Hotel.new("d3256870")
+
+	it "has name '北京怡亨酒店'" do 
+		expect(bj_sample.name).to eq '北京怡亨酒店'
+	end
+
+	it "has stars 5.0星级" do
+		expect(bj_sample.stars).to eq 5
+	end
+
+	it "has locality '北京市'" do
+		expect(bj_sample.locality).to eq "北京市"
+	end
+
+	it "has country name '中国' " do
+		expect(bj_sample.country).to eq "中国"
+	end
+
+
+	it "has rank '1'" do
+		expect(bj_sample.rank).to eq 1
+	end
+
+	it "has desc '北京怡亨酒店位处世界最繁忙的首都之一,共设100间房间,酒店细节专为商务旅客而设计。26间客房和套房,均于浴室内设置旋流浴,其中20间奢华套房更特设私人户外游泳池。房间内提供我们著名的“怡亨生活精品”,为高级住宿体验带来新定义。怡亨酒店不仅让您细味现代奢华的艺术,亦让您品味现代艺术的奢华。' " do
+		expect(bj_sample.desc).to eq "北京怡亨酒店位处世界最繁忙的首都之一,共设100间房间,酒店细节专为商务旅客而设计。26间客房和套房,均于浴室内设置旋流浴,其中20间奢华套房更特设私人户外游泳池。房间内提供我们著名的“怡亨生活精品”,为高级住宿体验带来新定义。怡亨酒店不仅让您细味现代奢华的艺术,亦让您品味现代艺术的奢华。"
+  end
+
+end

data/spec/rank_fast_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+describe Daodao do
+  ranks = Daodao.rank('http://www.daodao.com/Hotel_Review-g298555-d2682464-Reviews-Four_Seasons_Hotel_Guangzhou-Guangzhou_Guangdong.html')
+  it "The first Ota name 'Booking.com' " do
+    expect(ranks[:rank1]).to eq 'Booking.com'
+  end
+end

data/spec/rank_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+describe Daodao::Rank do
+  gz_sample = Daodao::Rank.new('/Hotel_Review-g298555-d2682464-Reviews-Four_Seasons_Hotel_Guangzhou-Guangzhou_Guangdong.html')
+  details = gz_sample.rooms_details
+  p gz_sample.rank
+  # screen shot
+  gz_sample.screen_shot
+
+  it "has otas amount '5'" do
+    expect(details.size).to eq 5
+  end
+
+  it "has lowest price '1903' " do
+    expect(details[2][:rooms_details][0][:room_price]).to eq 1903
+  end
+
+  it "The first Ota name 'Booking.com' " do
+    expect(details[0][:ota_name]).to eq 'Booking.com'
+  end
+
+  it "The rank 3 ota'name '艺龙旅行网' " do
+    expect(details[2][:ota_name]).to eq '艺龙旅行网'
+  end
+
+  it "The rooms amount provided by elong '10'" do
+    expect(details[2][:rooms_details].size).to eq 10
+  end
+  
+  it "The highest price of the rooms '4230'" do
+    expect(details[0][:rooms_details][3][:room_price]).to eq 4230
+  end
+
+  it "The room-type with the lowest price '高级城景房'" do
+    expect(details[2][:rooms_details][0][:room_type]).to eq '高级城景房'
+  end
+
+  it "The first room's price of the last ota '2330'"do
+    expect(details[4][:rooms_details][0][:room_price]).to eq 2330
+  end 
+
+  it "has The hotel name '四季酒店'"do
+    expect(gz_sample.name).to eq "四季酒店"
+  end 
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,91 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    # be_bigger_than(2).and_smaller_than(4).description
+    #   # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #   # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Limits the available syntax to the non-monkey patched syntax that is recommended.
+  # For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end
+
+require 'daodao'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daodao
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.3
 platform: ruby
 authors:
-- charles
+- timlen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-25 00:00:00.000000000 Z
+date: 2015-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,21 +38,44 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-description: daodao hotel list
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: daodao hotel list.
 email:
-- charles.liu@corp.elong.com
+- tinglenxan@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - daodao.gemspec
 - lib/daodao.rb
+- lib/daodao/hotel.rb
+- lib/daodao/hotel_list.rb
+- lib/daodao/rank.rb
 - lib/daodao/version.rb
+- spec/hotel_list_spec.rb
+- spec/hotel_spec.rb
+- spec/rank_fast_spec.rb
+- spec/rank_spec.rb
+- spec/spec_helper.rb
 homepage: ''
 licenses:
 - MIT
@@ -73,8 +96,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.4
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: daodao hotel list
-test_files: []
+summary: ''
+test_files:
+- spec/hotel_list_spec.rb
+- spec/hotel_spec.rb
+- spec/rank_fast_spec.rb
+- spec/rank_spec.rb
+- spec/spec_helper.rb