scrapers 1.5.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36a2d6b56890525cfb298805f2a0257c8873d902
-  data.tar.gz: 557b950125f8ef2ee96f0f007aba64ac5db87f04
+  metadata.gz: fd5f3c7ae7ee4f4730897d6ae083e5f2fc00f93f
+  data.tar.gz: b8fbe17e0456534b91e70d406550ea6ad8daff67
 SHA512:
-  metadata.gz: 57937d86734ba2850fdf80d7a533ec45899b4461838d351f00cb3bbaa01dff8e48a693dffd1fdba66f5b8ba63898385226d909061650e8e4f71ccc34ca81c3dd
-  data.tar.gz: 5a8080f12716288ec2af5ae7ff99e4a06c924c87cad151bb54a340b49277224e4d74bf7d7367996580adbbbaa15aff46fe76092b976fb52f6b685240a3ebaa79
+  metadata.gz: 24e3339257c087aaad5ef86b9d618c7d71bb7145ea3f36cf672abc850eba581d0c2c3ddb7e077b4e5fcfdce2f4db924091c1ba0c677997f50c5072089e10adae
+  data.tar.gz: fb7e81cd96b2c42065256b29adc1a767f8998e7e5261b59152a4cf77a2c2878e07ff7b6cefb6025a8f27f7da4512960a8f1ea0996d9e7866ac7d4ef8a235235b

data/Gemfile

@@ -5,6 +5,7 @@ gemspec
 
 group :development, :test do
   gem "pry"
+  gem "pry-byebug"
   gem "pry-nav"
   gem "pry-rescue"
   gem "pry-stack_explorer"

data/bin/new_scraper

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 require 'thor'
-require 'scrapers'
+# require 'scrapers'
 require "active_support/core_ext/string/inflections"
 require 'pry'
 
@@ -8,26 +8,28 @@ class NewScraper < Thor::Group
   include Thor::Actions
   
   argument :name
-  class_option :test_framework, :aliases => "-T", :default => :spec
-  
+  class_option :make_bin, :aliases => "-B", :default => true
+    
   def self.source_root
     File.expand_path("../../", __FILE__)
   end
 
-
-  def create_lib_file
-    template("templates/new_scraper.tt", "lib/scrapers/#{name.underscore}.rb")
+  def create_bin_file
+    return unless options["make_bin"]
+    template("templates/bin/new_scraper_bin.erb", "bin/#{file_name}")
+    template("templates/lib/new_scraper_thor.rb.erb", "lib/#{file_name}_thor.rb")
+    template("templates/spec/new_scraper_thor_spec.rb.erb", "spec/#{file_name}_thor_spec.rb")
   end
 
-  def create_test_file
+  def create_scraper_lib_file
+    template("templates/lib/scrapers/new_scraper_lib.rb.erb", "lib/scrapers/#{file_name}.rb")
+    template("templates/spec/lib/new_scraper_lib_spec.rb.erb", "spec/lib/scrapers/#{file_name}_spec.rb")
+  end
 
-    if %w[test_unit minitest test].include?(options["test_framework"])
-      test = "test"
-    else
-      test = "spec"
-    end
+  private
 
-    template("templates/new_scraper_#{test}.tt", "#{test}/scrapers/#{name.underscore}_#{test}.rb")
+  def file_name
+    name.underscore
   end
 
 end

data/bin/rubytapas

@@ -60,7 +60,11 @@ class RubyTapasDownload < Thor
     showlist_urls = Scrapers::RubyTapas.showlist(url, user, pw)
 
     showlist_urls.each do |url|
-      Scrapers::RubyTapas.scrape url, user, pw, destination
+      begin
+        Scrapers::RubyTapas.scrape url, user, pw, destination
+      rescue Errno::EEXIST
+        puts "episode exists, skipping"
+      end
       print "pausing..."
       sleep 5
       puts "."

data/bin/wunderground

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require 'wunderground_thor'
+
+result = Wunderground.start(ARGV)
+
+if result.is_a?(Hash) && result.has_key?("location")
+   puts <<-EOT
+Current conditions for: #{result["location"]}:
+#{"=" * 80}
+Temperature: #{result["temp"]}
+#{result["condition"]}, Feels like #{result["feel"]}
+High today: #{result["high"]} Low: #{result["low"]}
+EOT
+else
+  puts result
+end

data/lib/scrapers/version.rb

@@ -1,9 +1,9 @@
 module Scrapers
   module Version
 
-    MAJOR = 1
-    MINOR = 5
-    BUILD = 6
+    MAJOR = 2
+    MINOR = 0
+    BUILD = 0
 
   end
   

data/lib/scrapers/wunderground.rb

@@ -0,0 +1,30 @@
+# -*- ruby -*-
+require 'mechanize'
+require 'pry'
+
+module Scrapers
+  module Wunderground
+
+    def self.scrape(url)
+      results = Hash.new
+
+      Mechanize.start do |m|
+        m.get(url)
+        results["title"] = m.page.title
+        results["url"] = m.page.uri.to_s
+        snippet = m.page.search('#weather-snippet')
+        results["snippet"] = snippet.to_html.gsub(/[\n\t]+/,'')
+        results["location"] = snippet.at_css('h1').text.strip
+        results["image"] = snippet.at_css('#condition-img div img')['src']
+        results["temp"] = snippet.at_css('#temp').text.gsub(/[[:space:]]+/,' ').strip
+        results["condition"] = snippet.at_css('#condition').text.strip
+        results["feel"] = snippet.at_css('#feel').text.gsub(/[[:space:]]+/,' ').strip
+        results["high"] = snippet.at_css('.high').text.gsub(/[[:space:]]+/,' ').strip
+        results["low"] = snippet.at_css('.low').text.gsub(/[[:space:]]+/,' ').strip
+      end
+
+      results
+    end
+
+  end
+end

data/lib/wunderground_thor.rb

@@ -0,0 +1,56 @@
+require 'thor'
+require 'scrapers/wunderground'
+require 'awesome_print'
+
+class Wunderground < Thor
+
+  VERSION = '1.0.0'
+  WUNDERGROUND_URL = "http://www.wunderground.com/"
+
+  desc "current", "Show the current local weather conditions from #{WUNDERGROUND_URL}"
+  def current
+    current_conditions
+  end
+
+  desc "location", "Show location for current local weather"
+  def location
+    current_conditions["location"]
+  end
+
+  desc "image", "URL for current local weather conditions"
+  def image
+    current_conditions["image"]
+  end
+
+  desc "temp", "Current local temperature with units"
+  def temp
+    current_conditions["temp"]
+  end
+
+  desc "condition", "Current condition"
+  def condition
+    current_conditions["condition"]
+  end
+
+  desc "feel", "Feels like temperature"
+  def feel
+    current_conditions["feel"]
+  end
+
+  desc "high", "Forecast High Temperature"
+  def high
+    current_conditions["high"]
+  end
+
+  desc "low", "Forecast Low temperature"
+  def low
+    current_conditions["low"]
+  end
+
+  private
+
+  def current_conditions
+    @_current_conditions ||= Scrapers::Wunderground.scrape(WUNDERGROUND_URL)
+  end
+
+end

data/spec/lib/scrapers/wunderground_spec.rb

@@ -0,0 +1,20 @@
+# -*- ruby -*-
+require 'spec_helper'
+require 'scrapers/wunderground'
+
+module Scrapers
+
+  describe Wunderground do
+    it{should respond_to :scrape}
+    context "scraping" do
+      before(:all) do
+        @scrape = VCR.use_cassette('wunderground') do
+          @result = Scrapers::Wunderground.scrape
+        end
+      end
+      
+      it {expect(@result).to_not be_nil}
+      
+    end
+  end
+end

data/spec/wunderground_thor_spec.rb

@@ -0,0 +1,6 @@
+require 'spec_helper'
+require 'wunderground_thor'
+
+describe Wunderground do
+
+end

data/templates/bin/new_scraper_bin.erb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require '<%= file_name %>_thor'
+
+<%= name %>.start

data/templates/lib/new_scraper_thor.rb.erb

@@ -0,0 +1,7 @@
+require 'thor'
+require 'scrapers/<%= file_name %>'
+
+class <%= name %> < Thor
+
+
+end

data/templates/{new_scraper.tt → lib/scrapers/new_scraper_lib.rb.erb}

@@ -7,7 +7,8 @@ module Scrapers
     def self.scrape(url)
       results = Hash.new
 
-      Mechanize.start(url) do |m|
+      Mechanize.start do |m|
+        m.get(url)
 
       end
 

data/templates/{new_scraper_spec.tt → spec/lib/new_scraper_lib_spec.rb.erb}

@@ -1,6 +1,6 @@
 # -*- ruby -*-
 require 'spec_helper'
-require 'scrapers/<%= name.underscore %>'
+require 'scrapers/<%= file_name %>'
 
 module Scrapers
 
@@ -8,7 +8,7 @@ module Scrapers
     it{should respond_to :scrape}
     context "scraping" do
       before(:all) do
-        @comic = VCR.use_cassette('<%= name.underscore %>') do
+        @scrape = VCR.use_cassette('<%= file_name %>') do
           @result = Scrapers::<%= name %>.scrape
         end
       end

data/templates/spec/new_scraper_thor_spec.rb.erb

@@ -0,0 +1,6 @@
+require 'spec_helper'
+require '<%= file_name %>_thor'
+
+describe <%= name %> do
+
+end

metadata

@@ -1,181 +1,181 @@
 --- !ruby/object:Gem::Specification
 name: scrapers
 version: !ruby/object:Gem::Version
-  version: 1.5.6
+  version: 2.0.0
 platform: ruby
 authors:
 - Tamara Temple
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-28 00:00:00.000000000 Z
+date: 2014-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: highline
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: A library of web site scrapers utilizing mechanize and other goodies.
@@ -186,11 +186,12 @@ executables:
 - manning_books
 - new_scraper
 - rubytapas
+- wunderground
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec-example
+- ".gitignore"
+- ".rspec-example"
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -199,6 +200,7 @@ files:
 - bin/manning_books
 - bin/new_scraper
 - bin/rubytapas
+- bin/wunderground
 - lib/scrapers.rb
 - lib/scrapers/allrecipes.rb
 - lib/scrapers/discoverynews.rb
@@ -212,8 +214,11 @@ files:
 - lib/scrapers/rubytapas.rb
 - lib/scrapers/sinfest.rb
 - lib/scrapers/version.rb
+- lib/scrapers/wunderground.rb
 - lib/scrapers/xkcd.rb
+- lib/wunderground_thor.rb
 - scrapers.gemspec
+- spec/lib/scrapers/wunderground_spec.rb
 - spec/scrapers/allrecipes_spec.rb
 - spec/scrapers/discoverynews_spec.rb
 - spec/scrapers/download_spec.rb
@@ -225,8 +230,12 @@ files:
 - spec/scrapers/xkcd_spec.rb
 - spec/scrapers_spec.rb
 - spec/spec_helper.rb
-- templates/new_scraper.tt
-- templates/new_scraper_spec.tt
+- spec/wunderground_thor_spec.rb
+- templates/bin/new_scraper_bin.erb
+- templates/lib/new_scraper_thor.rb.erb
+- templates/lib/scrapers/new_scraper_lib.rb.erb
+- templates/spec/lib/new_scraper_lib_spec.rb.erb
+- templates/spec/new_scraper_thor_spec.rb.erb
 - vcr_cassettes/allrecipes_morning-glory-muffins-i.yml
 - vcr_cassettes/disconews_history-of-space.yml
 - vcr_cassettes/download-newfile.yml
@@ -246,21 +255,22 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Web site scrapers
 test_files:
+- spec/lib/scrapers/wunderground_spec.rb
 - spec/scrapers/allrecipes_spec.rb
 - spec/scrapers/discoverynews_spec.rb
 - spec/scrapers/download_spec.rb
@@ -272,3 +282,4 @@ test_files:
 - spec/scrapers/xkcd_spec.rb
 - spec/scrapers_spec.rb
 - spec/spec_helper.rb
+- spec/wunderground_thor_spec.rb