wayback_archiver 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa04295bda99658571157a5b1e21c5c43ffa9a98
-  data.tar.gz: 40906449581c37b2c17990a69d58a27a971c8ed7
+  metadata.gz: e0c99c305a785b125b14194417d2930a460b4a95
+  data.tar.gz: 4dc88ae01e5cdadfc5730aed1091f005d7526e58
 SHA512:
-  metadata.gz: af3d03dcc5781889c1fa10bd1a0851e053ab3e5d91acbd87aa08bebe3f408103ed1acff2a33e95004f5734ec24ae7c06d11d65003cfde411a16453d21920136f
-  data.tar.gz: f799a26ef78e340e3af0e31fd333917fb8723cd9a50eb29e12c69427591988a58cbb4bb2505c0027ddfa864604a3c8f8a226668ea98c1eab1abf6ebd6a7dbf73
+  metadata.gz: 1c665d9a1350cbbf8917803a3f8d0a6304a94b4af364497b3520dfe22b526c19ec9ae6174bbe4699c6d0dffd679427f457fb404757c4720d4d642de21afa465d
+  data.tar.gz: 5105b2fef3872e3dc528cac943f43c3e3c1b6007838bbebb6064a9f4351f8c44f306976a8e9977579aad8e6da642bf39a56b0b53de128ca5b0d8f89552c5c2c1

data/lib/wayback_archiver.rb

@@ -43,15 +43,15 @@ module WaybackArchiver
   #    WaybackArchiver.archive('example.com', strategy: :url, concurrency: 10)
   #    WaybackArchiver.archive('example.com', strategy: :url, limit: 100) # send max 100 URLs
   #    WaybackArchiver.archive('example.com', :url)
-  def self.archive(source, legacy_strategy = nil, strategy: :auto, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit)
+  def self.archive(source, legacy_strategy = nil, strategy: :auto, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
     strategy = legacy_strategy || strategy
 
     case strategy.to_s
-    when 'crawl'   then crawl(source, concurrency: concurrency, limit: limit)
-    when 'auto'    then auto(source, concurrency: concurrency, limit: limit)
-    when 'sitemap' then sitemap(source, concurrency: concurrency, limit: limit)
-    when 'urls'    then urls(source, concurrency: concurrency, limit: limit)
-    when 'url'     then urls(source, concurrency: concurrency, limit: limit)
+    when 'crawl'   then crawl(source, concurrency: concurrency, limit: limit, &block)
+    when 'auto'    then auto(source, concurrency: concurrency, limit: limit, &block)
+    when 'sitemap' then sitemap(source, concurrency: concurrency, limit: limit, &block)
+    when 'urls'    then urls(source, concurrency: concurrency, limit: limit, &block)
+    when 'url'     then urls(source, concurrency: concurrency, limit: limit, &block)
     else
       raise ArgumentError, "Unknown strategy: '#{strategy}'. Allowed strategies: sitemap, urls, url, crawl"
     end
@@ -69,11 +69,11 @@ module WaybackArchiver
   # @example Auto archive example.com and archive max 100 URLs
   #    WaybackArchiver.auto('example.com', limit: 100)
   # @see http://www.sitemaps.org
-  def self.auto(source, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit)
+  def self.auto(source, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
     urls = Sitemapper.autodiscover(source)
-    return urls(urls, concurrency: concurrency) if urls.any?
+    return urls(urls, concurrency: concurrency, &block) if urls.any?
 
-    crawl(source, concurrency: concurrency)
+    crawl(source, concurrency: concurrency, &block)
   end
 
   # Crawl site for URLs to send to the Wayback Machine.
@@ -86,9 +86,9 @@ module WaybackArchiver
   #    WaybackArchiver.crawl('example.com', concurrency: 1)
   # @example Crawl example.com and archive max 100 URLs
   #    WaybackArchiver.crawl('example.com', limit: 100)
-  def self.crawl(url, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit)
+  def self.crawl(url, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
     WaybackArchiver.logger.info "Crawling #{url}"
-    Archive.crawl(url, concurrency: concurrency, limit: limit)
+    Archive.crawl(url, concurrency: concurrency, limit: limit, &block)
   end
 
   # Get URLs from sitemap and send found URLs to the Wayback Machine.
@@ -102,9 +102,9 @@ module WaybackArchiver
   # @example Get example.com sitemap archive max 100 URLs
   #    WaybackArchiver.sitemap('example.com/sitemap.xml', limit: 100)
   # @see http://www.sitemaps.org
-  def self.sitemap(url, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit)
+  def self.sitemap(url, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
     WaybackArchiver.logger.info "Fetching Sitemap"
-    Archive.post(URLCollector.sitemap(url), concurrency: concurrency, limit: limit)
+    Archive.post(URLCollector.sitemap(url), concurrency: concurrency, limit: limit, &block)
   end
 
   # Send URL to the Wayback Machine.
@@ -117,8 +117,8 @@ module WaybackArchiver
   #    WaybackArchiver.urls(%w(example.com google.com))
   # @example Archive example.com, max 100 URLs
   #    WaybackArchiver.urls(%w(example.com www.example.com), limit: 100)
-  def self.urls(urls, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit)
-    Archive.post(Array(urls), concurrency: concurrency)
+  def self.urls(urls, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
+    Archive.post(Array(urls), concurrency: concurrency, &block)
   end
 
   # Set logger
@@ -180,4 +180,21 @@ module WaybackArchiver
   def self.max_limit
     @max_limit ||= DEFAULT_MAX_LIMIT
   end
+
+  # Sets the adapter
+  # @return [Object, #call>] the configured adapter
+  # @param [Object, #call>] the adapter
+  def self.adapter=(adapter)
+    unless adapter.respond_to?(:call)
+      raise(ArgumentError, 'adapter must implement #call')
+    end
+
+    @adapter = adapter
+  end
+
+  # Returns the configured adapter
+  # @return [Integer] the configured or the default adapter
+  def self.adapter
+    @adapter ||= WaybackMachine
+  end
 end

data/lib/wayback_archiver/adapters/wayback_machine.rb

@@ -0,0 +1,30 @@
+require 'wayback_archiver/archive_result'
+require 'wayback_archiver/request'
+
+module WaybackArchiver
+  # WaybackMachine adapter
+  class WaybackMachine
+    # Wayback Machine base URL.
+    BASE_URL    = 'https://web.archive.org/save/'.freeze
+
+    # Send URL to Wayback Machine.
+    # @return [ArchiveResult] the sent URL.
+    # @param [String] url to send.
+    # @example Archive example.com, with default options
+    #    WaybackMachine.call('http://example.com')
+    def self.call(url)
+      request_url  = "#{BASE_URL}#{url}"
+      response = Request.get(request_url, follow_redirects: false)
+      WaybackArchiver.logger.info "Posted [#{response.code}, #{response.message}] #{url}"
+      ArchiveResult.new(
+        url,
+        code: response.code,
+        request_url: response.uri,
+        response_error: response.error
+      )
+    rescue Request::Error => e
+      WaybackArchiver.logger.error "Failed to archive #{url}: #{e.class}, #{e.message}"
+      ArchiveResult.new(url, error: e)
+    end
+  end
+end

data/lib/wayback_archiver/archive.rb

@@ -1,15 +1,11 @@
 require 'concurrent'
 
-require 'wayback_archiver/archive_result'
 require 'wayback_archiver/thread_pool'
-require 'wayback_archiver/request'
+require 'wayback_archiver/adapters/wayback_machine'
 
 module WaybackArchiver
   # Post URL(s) to Wayback Machine
   class Archive
-    # Wayback Machine base URL.
-    WAYBACK_BASE_URL    = 'https://web.archive.org/save/'.freeze
-
     # Send URLs to Wayback Machine.
     # @return [Array<ArchiveResult>] with sent URLs.
     # @param [Array<String>] urls to send to the Wayback Machine.
@@ -97,18 +93,7 @@ module WaybackArchiver
     # @example Archive example.com, with default options
     #    Archive.post_url('http://example.com')
     def self.post_url(url)
-      request_url  = "#{WAYBACK_BASE_URL}#{url}"
-      response = Request.get(request_url, follow_redirects: false)
-      WaybackArchiver.logger.info "Posted [#{response.code}, #{response.message}] #{url}"
-      ArchiveResult.new(
-        url,
-        code: response.code,
-        request_url: response.uri,
-        response_error: response.error
-      )
-    rescue Request::Error => e
-      WaybackArchiver.logger.error "Failed to archive #{url}: #{e.class}, #{e.message}"
-      ArchiveResult.new(url, error: e)
+      WaybackArchiver.adapter.call(url)
     end
   end
 end

data/lib/wayback_archiver/archive_result.rb

@@ -16,6 +16,11 @@ module WaybackArchiver
       uri
     end
 
+    # @return [Boolean] true if success
+    def success?
+      !errored?
+    end
+
     # @return [Boolean] true if errored
     def errored?
       !!error

data/lib/wayback_archiver/version.rb

@@ -1,4 +1,4 @@
 module WaybackArchiver
   # Gem version
-  VERSION = '1.1.1'.freeze
+  VERSION = '1.2.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-06 00:00:00.000000000 Z
+date: 2017-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr
@@ -175,6 +175,7 @@ extra_rdoc_files: []
 files:
 - bin/wayback_archiver
 - lib/wayback_archiver.rb
+- lib/wayback_archiver/adapters/wayback_machine.rb
 - lib/wayback_archiver/archive.rb
 - lib/wayback_archiver/archive_result.rb
 - lib/wayback_archiver/http_code.rb