upton 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 892592f6c890ecd94fb1bdf3b8cc500e813ebfa3
-  data.tar.gz: 95d10ea4c37aaec611c76dc98c45dd449a1ac35d
+  metadata.gz: ff34d4a9c4a6356b1018dd6a754ceba17c42cf7d
+  data.tar.gz: d8f78bf65235f590795c9ad88ad6327ea096a7d4
 SHA512:
-  metadata.gz: f112a48ed90264ac5e111e48b45e6b67468793059f613385faa87bd6ab5122a7f11c532358daf7382c689e42b66db022bfce24ffd0b32ffe15619de0a026df77
-  data.tar.gz: 8fcbd1276ea6e284481d0395de5a1f73c07a2acb159edab6775c875d3dbf76e2b3c16c6b46b2aa48aec345fbd4950a5af179a6e560a1af5923e5087a8c6a648b
+  metadata.gz: bf4fa0283c1902ddf7cdb73762580ce4cb7477d1232e3d71ea22d1262c68bb019c064b98a8fd3d3e3d1cd61cf891c018ad66f9123ff0e8f9350e96b3a90217ac
+  data.tar.gz: ee02d8805a78d7332aac844d7ce486083debab43443ac702eabccf4349018907ce8001d2c755c1cbf23ea6048f631d0d9f18728b47e09fddb5435bb2230c136e

data/lib/upton.rb

@@ -46,6 +46,7 @@ module Upton
     ##
     def scrape(&blk)
       self.url_array = self.get_index unless self.url_array
+      blk = Proc.new{|x| x} if blk.nil?
       self.scrape_from_list(self.url_array, blk)
     end
 
@@ -146,7 +147,7 @@ module Upton
     #
     ##
     def next_index_page_url(url, pagination_index)
-      return EMPTY_STRING unless @paginated
+      return url unless @paginated
 
       if pagination_index > @pagination_max_pages
         puts "Exceeded pagination limit of #{@pagination_max_pages}" if @verbose
@@ -291,17 +292,19 @@ module Upton
     # Returns the concatenated output of each member of a paginated index,
     # e.g. a site listing links with 2+ pages.
     ##
-    def get_index_pages(url, pagination_index, pagination_interval, options={})
-      resps = [self.get_page(url, @index_debug, options)]
-      prev_url = url
-      while !resps.last.empty?
-        pagination_index += pagination_interval
-        next_url = self.next_index_page_url(url, pagination_index)
-        next_url = resolve_url(next_url, url)
-        break if next_url == prev_url || next_url.empty?
+    def get_index_pages(original_url, pagination_index, pagination_interval, options={})
+      resps = []
+      prev_url = nil
+      while resps.empty? || !resps.last.empty?
+        next_url = self.next_index_page_url(original_url, pagination_index)
+        break if next_url.empty?
+        
+        next_url = resolve_url(next_url, original_url)
+        break if next_url == prev_url
 
         next_resp = self.get_page(next_url, @index_debug, options).to_s
         prev_url = next_url
+        pagination_index += pagination_interval
         resps << next_resp
       end
       resps

data/lib/upton/downloader.rb

@@ -103,7 +103,7 @@ module Upton
       msg = "Stashed file retrieved by Upton #{Upton::VERSION} from #{@uri} at #{Time.now}"
       resp_html = Nokogiri::HTML(resp)
       comment = Nokogiri::XML::Comment.new(resp_html, msg)
-      if resp_html.root.nil?
+      if resp_html.root.nil? || !resp_html.include?("<html")
         return resp
       elsif resp_html.root.children.empty?
         resp_html.root.add_child(comment)

data/lib/upton/version.rb

@@ -1,3 +1,3 @@
 module Upton # :nodoc:
-    VERSION = '0.3.1'
+    VERSION = '0.3.3'
 end

data/spec/upton_spec.rb

@@ -139,6 +139,8 @@ describe Upton do
   it "should scrape paginated pages" do
     stub_request(:get, "www.example.com/propublica_search.html").
       to_return(:body => File.new('./spec/data/propublica_search.html'), :status => 200)
+    stub_request(:get, "www.example.com/propublica_search.html?p=1").
+      to_return(:body => File.new('./spec/data/propublica_search.html'), :status => 200)
     stub_request(:get, "www.example.com/propublica_search.html?p=2").
       to_return(:body => File.new('./spec/data/propublica_search_page_2.html'), :status => 200)
     stub_request(:get, "www.example.com/propublica_search.html?p=3").
@@ -197,6 +199,8 @@ describe Upton do
     FileUtils.rm_r("test_stashes") if Dir.exists?("test_stashes")
     stub_request(:get, "www.example.com/propublica_search.html").
       to_return(:body => File.new('./spec/data/propublica_search.html'), :status => 200)
+    stub_request(:get, "www.example.com/propublica_search.html?p=1").
+      to_return(:body => File.new('./spec/data/propublica_search.html'), :status => 200)
     stub_request(:get, "www.example.com/propublica_search.html?p=2").
       to_return(:body => File.new('./spec/data/propublica_search_page_2.html'), :status => 200)
     stub_request(:get, "www.example.com/propublica_search.html?p=3").

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upton
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Jeremy B. Merrill
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-16 00:00:00.000000000 Z
+date: 2014-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -66,20 +66,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.5.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -114,28 +100,14 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: mechanize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.1
 description: Don't re-write web scrapers every time. Upton gives you a scraper template
   that's easy to use for debugging and doesn't hammer servers by default.
 email: jeremybmerrill@jeremybmerrill.com