taiwan_tours 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2e64121574b9a2cbacc8da265b95fdd1921e9671
+  data.tar.gz: 3e585b8ba243438ee3b007ffd1d214c2d81f6a02
+SHA512:
+  metadata.gz: 55193252cc08899017ac0b8bad0157986b3efc452dcb5dc2789b1522425c513723c3fbdc1fede4a1a8568d18b2f889dcc1a44e9d9febe8070dbba6efa1b072fc
+  data.tar.gz: f7109e3c3f637f9598dc1bc8dda6c80efdce811a2d48817ef8cae7aecd949734ef1efdc1ae39760e18d776d46afad287efd7409c8c345fe217f2af8d06a1e855

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+rvm:
+ - ruby-head
+ - "2.2.3"
+ - "2.1.0"
+ - "1.9.3"
+ - jruby-head
+ - jruby-19mode
+matrix:
+ allow_failures:
+  - rvm: jruby-head
+branches:
+ only:
+  - master
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Gems required
+gem 'oga'
+gem 'json'
+gem 'minitest'
+gem 'webmock'  
+gem 'vcr'
+
+group :setup do
+ gem 'rake'
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2015 ZhongMeiZhou https://github.com/ZhongMeiZhou
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# taiwan_tours [![Build Status](https://travis-ci.org/ZhongMeiZhou/scraper_project.svg)](https://travis-ci.org/ZhongMeiZhou/scraper_project)
+
+ The lonelyplanet web scraper service traverses details of Taiwan tour packages in a easiest way and with simple steps.
+
+ Our company entirely respect the privacy policies of our partner Lonelyplanet declared on 'robots.txt' file.
+
+
+## Gem Usage
+
+ Install our fantastic gem using the following simple command on your command line:
+
+ ```sh
+ $ gem install taiwan_tours
+ ```
+
+ either, as our gem is published by Rubygems.org you can also include it in your 'Gemfile' as:
+
+ ```ruby
+ gem taiwan_tours
+ ```
+
+## Try it yourself!
+ Run it from the command line as follow:
+
+ ```sh
+ $ taiwan_tours
+ ```
+
+ or it can also be include it in your own Ruby code with the steps:
+
+```ruby
+require 'taiwan_tours'
+taiwan =  LonelyPlanetScrape::LonelyPlanetTours.new
+tours = taiwan.tours
+```
+
+## Want to make improvements?
+
+1. Fork it ( https://github.com/ZhongMeiZhou/scraper_project )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+task :default => [:spec]
+
+desc 'Run specs'
+Rake::TestTask.new(name=:spec) do |t|
+  t.pattern = 'spec/*_spec.rb'
+end

data/bin/taiwan_tours

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require 'json'
+require './lib/taiwan_tours/lonelyplanet_scrap'
+
+begin
+  taiwan_tours = LonelyPlanetScrape::LonelyPlanetTours.new
+  tour_arr = JSON.parse(taiwan_tours.tours)
+  puts 'According to LonelyPlanet, these are the best tour packages in Taiwan:'
+
+  tour_arr.each do |hash|
+    puts "- #{hash['title']} for $#{hash['price']}"
+  end
+
+rescue => e
+  puts "Error occured - see details: #{e}"
+end

data/lib/lonelyplanet_scrap_test.rb

@@ -0,0 +1,52 @@
+require 'oga'
+require 'open-uri'
+require 'json'
+require './lib/taiwan_tours/lonelyplanet_scrap'
+# Test class practice for handling errors and checking for HTML structure changes
+# can be used to test functionality before implementing it in Scraper Class
+class LonelyPlanetToursTest < LonelyPlanetScrape::LonelyPlanetTours
+  # override super class initialize function
+  def initialize
+    test_parse_html
+  end
+
+  def test_tour
+    @tours ||= test_html_extraction
+  end
+
+  private
+
+  # test connection to external site
+  def test_parse_html
+    parse_html
+  rescue OpenURI::HTTPError => e
+    puts "HTTP request error: #{e}"
+  end
+
+  # test scraping service and also test for changes in structure of elements being traversed
+  def test_html_extraction
+    result = []
+    fail 'OOPS, root article may have been changed or removed' if @document.xpath(TOUR_XPATH_CARD).text.empty?
+    fail 'Title h1 tag may have been changed or removed' if @document.xpath(CARD_TITLE_XPATH).text.empty?
+    fail 'Price span tag may have been changed or removed' if @document.xpath(CARD_PRICE_AMOUNT_XPATH).text.empty?
+    fail 'Content div tag may have been changed or removed' if @document.xpath(CARD_CONTENT_XPATH).text.empty?
+
+    @document.xpath(TOUR_XPATH_CARD).map do |card|
+      element = {}
+      element['img'] = card.xpath(CARD_IMGLINK_XPATH).text
+      element['title'] = card.xpath(CARD_TITLE_XPATH).text.strip
+      element['content'] = card.xpath(CARD_CONTENT_XPATH).text.strip
+      element['location'] = card.xpath(CARD_LOCATION_XPATH).text
+      element['price_currency'] = card.xpath(CARD_PRICE_CURRENCY_XPATH).text
+      element['price'] = card.xpath(CARD_PRICE_AMOUNT_XPATH).text
+      result << element
+    end
+    result.to_json
+  rescue StandardError => e
+    puts e.message
+    puts e.backtrace.inspect
+  end
+end
+
+test_run = LonelyPlanetToursTest.new
+puts test_run.test_tour

data/lib/taiwan_tours/lonelyplanet_scrap.rb

@@ -0,0 +1,52 @@
+require 'oga'
+require 'open-uri'
+require 'json'
+
+# Module defines LonelyPlanetTours class which handles scraping of lonelyplanet Taiwan tours page
+module LonelyPlanetScrape
+  class LonelyPlanetTours
+    # Define constants needed for scraping
+    # Please if the test for uri fails replace contant manually
+    LONELYPLANET_URL = 'http://www.lonelyplanet.com'
+    TOUR_RELATIVE_DIR = 'taiwan/tours'
+
+    TOUR_XPATH_CARD = "//article[contains(@class,'card')]"
+    CARD_IMGLINK_XPATH = ".//img[contains(@class,'card__figure__img')]/@src"
+    CARD_TITLE_XPATH = './/h1'
+    CARD_CONTENT_XPATH = ".//div[contains(@class,'card__content__desc')]//p"
+    CARD_LINK_XPATH = ".//div[contains(@class,'card__mask')]//a"
+    CARD_LOCATION_XPATH = ".//div[contains(@class,'card__footer__locale')]"
+    CARD_PRICE_CURRENCY_XPATH = ".//span[contains(@class,'js-currency')]"
+    CARD_PRICE_AMOUNT_XPATH = ".//span[contains(@class,'js-price')]"
+
+    def initialize
+      parse_html
+    end
+
+    def tours
+      @tours ||= extract_tours
+    end
+
+    private
+
+    def parse_html
+      url = "#{LONELYPLANET_URL}/#{TOUR_RELATIVE_DIR}"
+      @document = Oga.parse_html(open(url))
+    end
+
+    def extract_tours
+      result = []
+      @document.xpath(TOUR_XPATH_CARD).map do |card|
+        element = {}
+        element['img'] = card.xpath(CARD_IMGLINK_XPATH).text
+        element['title'] = card.xpath(CARD_TITLE_XPATH).text.strip
+        element['content'] = card.xpath(CARD_CONTENT_XPATH).text.strip
+        element['location'] = card.xpath(CARD_LOCATION_XPATH).text
+        element['price_currency'] = card.xpath(CARD_PRICE_CURRENCY_XPATH).text
+        element['price'] = card.xpath(CARD_PRICE_AMOUNT_XPATH).text
+        result << element
+      end
+      result.to_json
+    end
+  end
+end

data/lib/taiwan_tours/version.rb

@@ -0,0 +1,5 @@
+# Versioning
+module LonelyPlanetScrape
+  VERSION = '0.1.0'
+  DATE = '2015-10-16'
+end

data/lib/taiwan_tours.rb

@@ -0,0 +1,2 @@
+require 'taiwan_tours/lonelyplanet_scrap.rb'
+require 'taiwan_tours/version.rb'