doctor_scrape 0.0.5 → 0.0.6

data/doctor_scrape.gemspec

@@ -15,7 +15,9 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = DoctorScrape::VERSION
 
-  gem.add_dependency "mechanize", ["~> 2.3"]
+  # gem.add_dependency "mechanize", ["~> 2.3"]
+  gem.add_dependency "nokogiri", ["~> 1.5.0"]
+  gem.add_dependency "unwind", ["~> 0.9.6"]
   # gem.add_dependency "text", ["~> 1.0.3"]
 
   gem.add_development_dependency "rspec", ["~> 2.8.0"]
@@ -31,5 +33,4 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "hirb", ["~> 0.6.0"]
   gem.add_development_dependency "awesome_print", ["~> 1.0.2"]
   gem.add_development_dependency "simplecov", ["~> 0.6.0"]
-  # gem.add_development_dependency "psych", ["~> 1.2.2"]
 end

data/lib/doctor_scrape.rb

@@ -1,14 +1,13 @@
 # encoding: utf-8
 require 'doctor_scrape/version'
-require 'mechanize'
 require 'nokogiri'
 require 'ostruct'
 require 'open-uri'
+require 'unwind'
 
 module DoctorScrape
   autoload :Data,             'doctor_scrape/data'
   autoload :Search,           'doctor_scrape/search'
-  autoload :RedirectFollower, 'doctor_scrape/redirect_follower'
 
   module Scraper
     autoload :Base,    'doctor_scrape/scraper/base'

data/lib/doctor_scrape/search.rb

@@ -29,7 +29,7 @@ module DoctorScrape
       end
 
       def resolve_urls(urls)
-        urls.map { |url| RedirectFollower.new(url).resolve }
+        urls.map { |url| Unwind::RedirectFollower.resolve(url).final_url }
       end
 
       def resolve_scrapers(urls)

data/lib/doctor_scrape/version.rb

@@ -1,3 +1,3 @@
 module DoctorScrape
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/spec/search_spec.rb

@@ -89,9 +89,9 @@ describe DoctorScrape::Search do
     let(:resolved) { ["http://bitly.com", "http://example.com", "http://googl.com"] }
 
     it "returns array with resolved urls" do
-      resolvers = resolved.map { |r| double "resolver", :resolve => r }
+      resolvers = resolved.map { |r| double "resolver", :final_url => r }
       urls.each_with_index do |url, i|
-        DoctorScrape::RedirectFollower.should_receive(:new).with(url) { resolvers[i] }
+        Unwind::RedirectFollower.should_receive(:resolve).with(url) { resolvers[i] }
       end
 
       DoctorScrape::Search.resolve_urls(urls).should == resolved

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: doctor_scrape
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,33 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-28 00:00:00.000000000 Z
+date: 2012-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: mechanize
-  requirement: &70186292553660 !ruby/object:Gem::Requirement
+  name: nokogiri
+  requirement: &70186149359280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 1.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *70186292553660
+  version_requirements: *70186149359280
+- !ruby/object:Gem::Dependency
+  name: unwind
+  requirement: &70186149358760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.6
+  type: :runtime
+  prerelease: false
+  version_requirements: *70186149358760
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70186292553140 !ruby/object:Gem::Requirement
+  requirement: &70186149358280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +43,10 @@ dependencies:
         version: 2.8.0
   type: :development
   prerelease: false
-  version_requirements: *70186292553140
+  version_requirements: *70186149358280
 - !ruby/object:Gem::Dependency
   name: vcr
-  requirement: &70186292552660 !ruby/object:Gem::Requirement
+  requirement: &70186149357800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +54,10 @@ dependencies:
         version: 2.0.0.rc1
   type: :development
   prerelease: false
-  version_requirements: *70186292552660
+  version_requirements: *70186149357800
 - !ruby/object:Gem::Dependency
   name: webmock
-  requirement: &70186292552180 !ruby/object:Gem::Requirement
+  requirement: &70186149357320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - <
@@ -54,10 +65,10 @@ dependencies:
         version: '1.8'
   type: :development
   prerelease: false
-  version_requirements: *70186292552180
+  version_requirements: *70186149357320
 - !ruby/object:Gem::Dependency
   name: guard
-  requirement: &70186292551700 !ruby/object:Gem::Requirement
+  requirement: &70186149356840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +76,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70186292551700
+  version_requirements: *70186149356840
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &70186292551220 !ruby/object:Gem::Requirement
+  requirement: &70186149356360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +87,10 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *70186292551220
+  version_requirements: *70186149356360
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &70186292550740 !ruby/object:Gem::Requirement
+  requirement: &70186149355880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +98,10 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *70186292550740
+  version_requirements: *70186149355880
 - !ruby/object:Gem::Dependency
   name: rb-fsevent
-  requirement: &70186292550260 !ruby/object:Gem::Requirement
+  requirement: &70186149355400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +109,10 @@ dependencies:
         version: 0.9.0
   type: :development
   prerelease: false
-  version_requirements: *70186292550260
+  version_requirements: *70186149355400
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &70186292549780 !ruby/object:Gem::Requirement
+  requirement: &70186149354920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,10 +120,10 @@ dependencies:
         version: 0.9.8.2
   type: :development
   prerelease: false
-  version_requirements: *70186292549780
+  version_requirements: *70186149354920
 - !ruby/object:Gem::Dependency
   name: pry-doc
-  requirement: &70186292549300 !ruby/object:Gem::Requirement
+  requirement: &70186149354440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,10 +131,10 @@ dependencies:
         version: 0.4.0
   type: :development
   prerelease: false
-  version_requirements: *70186292549300
+  version_requirements: *70186149354440
 - !ruby/object:Gem::Dependency
   name: pry-editline
-  requirement: &70186292548820 !ruby/object:Gem::Requirement
+  requirement: &70186149353960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,10 +142,10 @@ dependencies:
         version: 1.1.1
   type: :development
   prerelease: false
-  version_requirements: *70186292548820
+  version_requirements: *70186149353960
 - !ruby/object:Gem::Dependency
   name: hirb
-  requirement: &70186292548340 !ruby/object:Gem::Requirement
+  requirement: &70186149353480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -142,10 +153,10 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *70186292548340
+  version_requirements: *70186149353480
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70186292547860 !ruby/object:Gem::Requirement
+  requirement: &70186149353000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -153,10 +164,10 @@ dependencies:
         version: 1.0.2
   type: :development
   prerelease: false
-  version_requirements: *70186292547860
+  version_requirements: *70186149353000
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70186292547380 !ruby/object:Gem::Requirement
+  requirement: &70186149352520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -164,7 +175,7 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *70186292547380
+  version_requirements: *70186149352520
 description: Library for scraping norwegian doctoral dissertations
 email:
 - gudleik@gmail.com
@@ -183,7 +194,6 @@ files:
 - doctor_scrape.gemspec
 - lib/doctor_scrape.rb
 - lib/doctor_scrape/data.rb
-- lib/doctor_scrape/redirect_follower.rb
 - lib/doctor_scrape/scraper/base.rb
 - lib/doctor_scrape/scraper/bora.rb
 - lib/doctor_scrape/scraper/diva.rb
@@ -214,7 +224,6 @@ files:
 - spec/parse/diva_spec.rb
 - spec/parse/duo_spec.rb
 - spec/parse/munin_spec.rb
-- spec/redirect_follower_spec.rb
 - spec/scraper_spec.rb
 - spec/scrapers/base_spec.rb
 - spec/scrapers/bora_spec.rb
@@ -240,7 +249,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4122406901322646134
+      hash: -2449353312367516367
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -249,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4122406901322646134
+      hash: -2449353312367516367
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.11
@@ -279,7 +288,6 @@ test_files:
 - spec/parse/diva_spec.rb
 - spec/parse/duo_spec.rb
 - spec/parse/munin_spec.rb
-- spec/redirect_follower_spec.rb
 - spec/scraper_spec.rb
 - spec/scrapers/base_spec.rb
 - spec/scrapers/bora_spec.rb

data/lib/doctor_scrape/redirect_follower.rb

@@ -1,29 +0,0 @@
-# encoding: utf-8
-require 'net/https'
-module DoctorScrape
-  class TooManyRedirects < StandardError; end
-
-  class RedirectFollower
-    attr_accessor :url
-
-    def initialize(url)
-      @url = url
-    end
-
-    def resolve(limit=5)
-      raise TooManyRedirects if limit == 0
-
-      response = Net::HTTP.get_response URI.parse(@url)
-
-      if response.is_a? Net::HTTPRedirection
-        @url = response['location']
-        resolve limit - 1
-      end
-
-      @url
-    rescue Net::HTTPBadResponse => error
-      # This can safely be ignored
-    end
-
-  end
-end

data/spec/redirect_follower_spec.rb

@@ -1,37 +0,0 @@
-# encoding: utf-8
-require 'spec_helper'
-
-describe DoctorScrape::RedirectFollower do
-  let(:url)      { "http://bit.ly/foobar" }
-  let(:endpoint) { "http://example.com"   }
-  let(:resolver) { DoctorScrape::RedirectFollower.new url }
-  before { stub_request(:any, endpoint).to_return(:body => "You found me!") }
-
-  context "when url doesn't redirect" do
-    before  { stub_request(:any, url).to_return(:body => "Ok") }
-    specify { resolver.resolve.should eq url }
-  end
-
-  context "when url redirects" do
-    before  { stub_request(:any, url).to_return(:status => [ 302, "Moved Temporarily" ], headers: { location: endpoint }) }
-    specify { resolver.resolve.should eq endpoint }
-  end
-
-  context "too many redirects" do
-    before  { stub_request(:any, url).to_return(:status => [ 302, "Moved Temporarily" ], headers: { location: url }) }
-
-    it "raises error after 5 redirects" do
-      expect { resolver.resolve }.to raise_error
-      a_request(:get, url).should have_been_made.times(5)
-    end
-  end
-
-  context "when exception occurs" do
-    it "returns the last url" do
-      stub_request(:get, url).to_return(:status => [ 302, "Moved Temporarily" ], headers: { location: endpoint })
-      stub_request(:get, endpoint).to_raise Net::HTTPBadResponse
-      resolver.resolve.should eq(endpoint)
-    end
-  end
-
-end