lonely_planet_tours 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 677cc5a7fdc5727128a5ff48e646da82ca1093ac
+  data.tar.gz: dc1b8c204fd73ebe75535688acfe4c6e67676e14
+SHA512:
+  metadata.gz: b20b00e93b82abf3b256adc8db5c56a51fd0210693b57b7e1f74a3a8f5269a697beb62d0cefed6b02654c18c65fff3d20d4bf8d0a71671705798c20d78584802
+  data.tar.gz: f9da365db0e0be8e142eb8cfcf8661b2f88881a40d43fc92eeedaa3e07140c3e0f11ffb578d5031f348f02d3c2d5efa860645a5b2d913571e6e81814e5b7c45b

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+rvm:
+ - ruby-head
+ - "2.2.3"
+ - "2.1.0"
+ - "1.9.3"
+ - jruby-head
+ - jruby-19mode
+matrix:
+ allow_failures:
+  - rvm: jruby-head
+branches:
+ only:
+  - master
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Gems required
+gem 'oga'
+gem 'json'
+gem 'minitest'
+gem 'webmock'  
+gem 'vcr'
+
+group :setup do
+ gem 'rake'
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2015 ZhongMeiZhou https://github.com/ZhongMeiZhou
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+
+
+# lonely_planet_tours [![Gem Version](https://badge.fury.io/rb/taiwan_tours.svg)](https://badge.fury.io/rb/taiwan_tours) [![Build Status](https://travis-ci.org/ZhongMeiZhou/scraper_project.svg)](https://travis-ci.org/ZhongMeiZhou/scraper_project)
+
+ The web scraper service we built provides listings of tour packages from Lonelyplanet based on a specified country. We have bundled this service into the lonely_planet_tours gem for you to use.
+
+ Note that we respect Lonelyplanet's 'robots.txt' file.
+
+
+## Usage
+
+ Install at the command line:
+
+ ```sh
+ $ gem install lonely_planet_tours
+ ```
+
+ or include it in your 'Gemfile' as:
+
+ ```ruby
+ gem lonely_planet_tours
+ ```
+
+## Try it yourself!
+ Run it from the command line as:
+
+ ```sh
+ $ lonely_planet_tours
+ ```
+
+ or include it in your code:
+
+```ruby
+require 'lonely_planet_tours'
+taiwan =  LonelyPlanetScrape::LonelyPlanetTours.new(country)
+tours = taiwan.tours
+```
+
+## Want to make improvements?
+
+1. Fork it ( https://github.com/ZhongMeiZhou/scraper_project )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+task :default => [:spec]
+
+desc 'Run specs'
+Rake::TestTask.new(name=:spec) do |t|
+  t.pattern = 'spec/*_spec.rb'
+end

data/bin/lonely_planet_tours

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'json'
+require '../lib/lonely_planet/lonelyplanet_scrap.rb'
+
+begin
+  fail ArgumentError, "usage: tours [country]\n" if ARGV.count == 0
+  country = ARGV[0]
+
+  taiwan_tours = LonelyPlanetScrape::LonelyPlanetTours.new(country)
+  tour_arr = JSON.parse(taiwan_tours.tours)
+  puts "According to LonelyPlanet, these are the best tour packages in #{country}:"
+
+  tour_arr.each do |hash|
+    puts "- #{hash['title']} for $#{hash['price']}"
+  end
+
+rescue => e
+  puts "Error occured - see details: #{e}"
+end

data/lib/lonely_planet/lonelyplanet_scrap.rb

@@ -0,0 +1,50 @@
+require 'oga'
+require 'open-uri'
+require 'json'
+
+# Module defines LonelyPlanetTours class which handles scraping of lonelyplanet Taiwan tours page
+module LonelyPlanetScrape
+  class LonelyPlanetTours
+    LONELYPLANET_URL = 'http://www.lonelyplanet.com'
+    TOUR_RELATIVE_DIR = 'tours'
+
+    TOUR_XPATH_CARD = "//article[contains(@class,'card')]"
+    CARD_IMGLINK_XPATH = ".//img[contains(@class,'card__figure__img')]/@src"
+    CARD_TITLE_XPATH = './/h1'
+    CARD_CONTENT_XPATH = ".//div[contains(@class,'card__content__desc')]//p"
+    CARD_LINK_XPATH = ".//div[contains(@class,'card__mask')]//a"
+    CARD_LOCATION_XPATH = ".//div[contains(@class,'card__footer__locale')]"
+    CARD_PRICE_CURRENCY_XPATH = ".//span[contains(@class,'js-currency')]"
+    CARD_PRICE_AMOUNT_XPATH = ".//span[contains(@class,'js-price')]"
+
+    def initialize(country)
+      parse_html(country)
+    end
+
+    def tours
+      @tours ||= extract_tours
+    end
+
+    private
+
+    def parse_html(country)
+      url = "#{LONELYPLANET_URL}/#{country}/#{TOUR_RELATIVE_DIR}"
+      @document = Oga.parse_html(open(url))
+    end
+
+    def extract_tours
+      result = []
+      @document.xpath(TOUR_XPATH_CARD).map do |card|
+        element = {}
+        element['img'] = card.xpath(CARD_IMGLINK_XPATH).text
+        element['title'] = card.xpath(CARD_TITLE_XPATH).text.strip
+        element['content'] = card.xpath(CARD_CONTENT_XPATH).text.strip
+        element['location'] = card.xpath(CARD_LOCATION_XPATH).text
+        element['price_currency'] = card.xpath(CARD_PRICE_CURRENCY_XPATH).text
+        element['price'] = card.xpath(CARD_PRICE_AMOUNT_XPATH).text
+        result << element
+      end
+      result.to_json
+    end
+  end
+end

data/lib/lonely_planet/version.rb

@@ -0,0 +1,5 @@
+# Versioning
+module LonelyPlanetScrape
+  VERSION = '1.0.0'
+  DATE = '2015-10-21'
+end

data/lib/lonely_planet_tours.rb

@@ -0,0 +1,2 @@
+require 'lonely_planet/lonelyplanet_scrap.rb'
+require 'lonely_planet/version.rb'

data/lib/lonelyplanet_scrap_test.rb

@@ -0,0 +1,52 @@
+require 'oga'
+require 'open-uri'
+require 'json'
+require './lib/lonely_planet/lonelyplanet_scrap'
+# Test class practice for handling errors and checking for HTML structure changes
+# can be used to test functionality before implementing it in Scraper Class
+class LonelyPlanetToursTest < LonelyPlanetScrape::LonelyPlanetTours
+  # override super class initialize function
+  def initialize(country)
+    test_parse_html(country)
+  end
+
+  def test_tour
+    @tours ||= test_html_extraction
+  end
+
+  private
+
+  # test connection to external site
+  def test_parse_html(country)
+    parse_html(country)
+  rescue OpenURI::HTTPError => e
+    puts "HTTP request error: #{e}"
+  end
+
+  # test scraping service and also test for changes in structure of elements being traversed
+  def test_html_extraction
+    result = []
+    fail 'OOPS, root article may have been changed or removed' if @document.xpath(TOUR_XPATH_CARD).text.empty?
+    fail 'Title h1 tag may have been changed or removed' if @document.xpath(CARD_TITLE_XPATH).text.empty?
+    fail 'Price span tag may have been changed or removed' if @document.xpath(CARD_PRICE_AMOUNT_XPATH).text.empty?
+    fail 'Content div tag may have been changed or removed' if @document.xpath(CARD_CONTENT_XPATH).text.empty?
+
+    @document.xpath(TOUR_XPATH_CARD).map do |card|
+      element = {}
+      element['img'] = card.xpath(CARD_IMGLINK_XPATH).text
+      element['title'] = card.xpath(CARD_TITLE_XPATH).text.strip
+      element['content'] = card.xpath(CARD_CONTENT_XPATH).text.strip
+      element['location'] = card.xpath(CARD_LOCATION_XPATH).text
+      element['price_currency'] = card.xpath(CARD_PRICE_CURRENCY_XPATH).text
+      element['price'] = card.xpath(CARD_PRICE_AMOUNT_XPATH).text
+      result << element
+    end
+    result.to_json
+  rescue StandardError => e
+    puts e.message
+    puts e.backtrace.inspect
+  end
+end
+
+test_run = LonelyPlanetToursTest.new('belize')
+puts test_run.test_tour

data/lonely_planet_tours.gemspec

@@ -0,0 +1,23 @@
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'lonely_planet/version'
+
+Gem::Specification.new do |s|
+  s.name        =  'lonely_planet_tours'
+  s.version     =  LonelyPlanetScrape::VERSION
+  s.date        =  LonelyPlanetScrape::DATE
+  s.executables << 'lonely_planet_tours'
+  s.summary     =  'The lonelyplanet web scraper service traverses details of tour packages based on specified country'
+  s.description =  'Get listings of tour packages from Lonelyplanet'
+  s.authors     =  ['Bayardo Salgado', 'Cesar Ordoñez','Eduardo Salazar','Nicole Weatherburne']
+  s.email       =  ['bayardo_salgado@yahoo.com','cesar','esalazar922@gmail.com','nikkiweat@gmail.com']
+  s.files       =  `git ls-files`.split("\n")
+  s.test_files  =  `git ls-files spec/*`.split("\n")
+  s.homepage    =  'https://github.com/ZhongMeiZhou/scraper_project'
+  s.license     =  'MIT'
+
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'webmock'
+  s.add_runtime_dependency 'oga'
+  s.add_runtime_dependency 'json'
+end