wayback_archiver 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e0c99c305a785b125b14194417d2930a460b4a95
-  data.tar.gz: 4dc88ae01e5cdadfc5730aed1091f005d7526e58
+SHA256:
+  metadata.gz: 1d97461d94b8ec02e1cae528f939be17991d054a1d29953c8672bfc8e29ea7cf
+  data.tar.gz: 75bf5c3a7214001df417678c7a084ea3bc1e80dc87773a8da17c1c1defcdaff8
 SHA512:
-  metadata.gz: 1c665d9a1350cbbf8917803a3f8d0a6304a94b4af364497b3520dfe22b526c19ec9ae6174bbe4699c6d0dffd679427f457fb404757c4720d4d642de21afa465d
-  data.tar.gz: 5105b2fef3872e3dc528cac943f43c3e3c1b6007838bbebb6064a9f4351f8c44f306976a8e9977579aad8e6da642bf39a56b0b53de128ca5b0d8f89552c5c2c1
+  metadata.gz: 00d2eef71ef692688249dc97cbaca906f6725679a19deab2fd3c5998e06319032ee9e9aff24bb68dd80254f1b18ef06d1aeadfeabd7fcaafa326f9bdcca10688
+  data.tar.gz: d05a72cfb6fbb2636f43c8d8737f72aa0587415a8ce344d1c1d01db9a3cdcecfc278d81d27e2962868dab0674e4ffdcc33fe9115f0969e8c32f1c490473e13aa

data/lib/wayback_archiver/request.rb

@@ -33,6 +33,19 @@ module WaybackArchiver
     # Max number of redirects before an error is raised
     MAX_REDIRECTS = 10
 
+    # Known request errors
+    REQUEST_ERRORS = {
+      # server
+      Timeout::Error => ServerError,
+      OpenSSL::SSL::SSLError => ServerError,
+      Net::HTTPBadResponse => ServerError,
+      Zlib::Error => ServerError,
+      # client
+      SystemCallError => ClientError,
+      SocketError => ClientError,
+      IOError => ClientError
+    }.freeze
+
     # Get reponse.
     # @return [Response] the http response representation.
     # @param [String, URI] uri to retrieve.
@@ -184,20 +197,11 @@ module WaybackArchiver
     private
 
     def self.perform_request(uri, http, request)
-      # TODO: Consider retrying failed requests
+      # TODO: Consider retrying on certain HTTP response codes, i.e 429, 503
       response = http.request(request)
       GETStruct.new(response)
-    rescue Timeout::Error,
-           OpenSSL::SSL::SSLError,
-           Net::HTTPBadResponse,
-           Zlib::Error => e
-
-      build_request_error(uri, e, ServerError)
-    rescue SystemCallError,
-           SocketError,
-           IOError => e
-
-      build_request_error(uri, e, ClientError)
+    rescue *REQUEST_ERRORS.keys => e
+      build_request_error(uri, e, REQUEST_ERRORS.fetch(e.class))
     end
 
     def self.build_request_error(uri, error, error_wrapper_klass)

data/lib/wayback_archiver/sitemapper.rb

@@ -1,3 +1,4 @@
+require 'set'
 require 'robots'
 
 require 'wayback_archiver/sitemap'
@@ -27,6 +28,7 @@ module WaybackArchiver
       WaybackArchiver.logger.info 'Looking for Sitemap(s) in /robots.txt'
       robots = Robots.new(WaybackArchiver.user_agent)
       sitemaps = robots.other_values(url)['Sitemap']
+
       if sitemaps
         return sitemaps.flat_map do |sitemap|
           WaybackArchiver.logger.info "Fetching Sitemap at #{sitemap}"
@@ -61,12 +63,21 @@ module WaybackArchiver
     # @example Get URLs defined in Sitemap
     #    Sitemapper.urls(xml: xml)
     # @see http://www.sitemaps.org
-    def self.urls(url: nil, xml: nil)
+    def self.urls(url: nil, xml: nil, visited: Set.new)
+      if visited.include?(url)
+        WaybackArchiver.logger.debug "Already visited #{url} skipping.."
+        return []
+      end
+
+      visited << url if url
+
       xml = Request.get(url).body unless xml
       sitemap = Sitemap.new(xml)
 
       if sitemap.sitemap_index?
-        sitemap.sitemaps.flat_map { |sitemap_url| urls(url: sitemap_url) }
+        sitemap.sitemaps.flat_map do |sitemap_url|
+          urls(url: sitemap_url, visited: visited)
+        end
       else
         sitemap.urls
       end

data/lib/wayback_archiver/version.rb

@@ -1,4 +1,4 @@
 module WaybackArchiver
   # Gem version
-  VERSION = '1.2.0'.freeze
+  VERSION = '1.2.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-20 00:00:00.000000000 Z
+date: 2018-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr
@@ -100,14 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -207,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Post URLs to Wayback Machine (Internet Archive)