wayback_archiver 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab2721984ab2e58857a30f83a871cc539248fc7d
-  data.tar.gz: e4a087f0d0500c930ae9e846dffb18bf72ec5696
+  metadata.gz: ccebbb815d374658a9b2e1b2998a40041115d295
+  data.tar.gz: 153afa895756670988fa3663f44fd1fd5a2e5e3e
 SHA512:
-  metadata.gz: 26cbd46ab21a7364b3ccd38ba7cd7405ed495eee635fc035a1b99a9addcdc9f670f662d1122ac7aa23ec691297da0e45f31083dc13681c1db74626f6705c0e67
-  data.tar.gz: c65fb8fd27f3dc9d58e33a501e66a79c24d807640c2ad3ce63a52e98e182ab46334fadf216adb3c20bdb176c1512232e7f1c4f0c0253a42d6559a21565809fab
+  metadata.gz: db7d655b1ea642618797d7fa3ece8357fa608dcb60322da76be7d793de11cd61df61f796025000ed0099edd96711aa78db607ce0614e9256e5921c252a7d931b
+  data.tar.gz: 919d8473c7f97bbd36c9065b95a99304ace7e0556a8dcd0f92eedf00d65961400dd92c63d86e405aa97babd5d254b522384f665ff39ababdc71afed9c13333ad

data/bin/wayback_archiver

@@ -5,6 +5,8 @@ require 'wayback_archiver'
 url  = ARGV[0]
 from = ARGV[1]
 
+WaybackArchiver.logger = Logger.new(STDOUT)
+
 if from.nil?
   WaybackArchiver.archive(url)
 else

data/lib/wayback_archiver/archive.rb

@@ -4,29 +4,45 @@ module WaybackArchiver
     # Wayback Machine base URL.
     WAYBACK_BASE_URL    = 'https://web.archive.org/save/'.freeze
     # Default concurrency for archiving URLs
-    DEFAULT_CONCURRENCY = 10
+    DEFAULT_CONCURRENCY = 5
     # Send URLs to Wayback Machine.
     # @return [Array] with sent URLs.
     # @param [Array] urls URLs to send.
     # @param [Hash] options
-    # @example Archive example.com, with default options
+    # @example Archive urls, asynchronously
     #    Archive.post(['http://example.com'])
-    # @example Archive example.com, using only 1 thread
+    # @example Archive urls, using only 1 thread
     #    Archive.post(['http://example.com'], concurrency: 1)
-    def self.post(urls, options = {})
-      options     = { concurrency: DEFAULT_CONCURRENCY }.merge!(options)
-      concurrency = options[:concurrency]
-
-      puts "=== WAYBACK ARCHIVER ==="
-      puts "Request are sent with up to #{concurrency} parallel threads"
-      puts "Total urls to be sent: #{urls.length}"
+    def self.post(urls, concurrency: DEFAULT_CONCURRENCY)
+      WaybackArchiver.logger.info "=== WAYBACK ARCHIVER ==="
+      WaybackArchiver.logger.info "Request are sent with up to #{concurrency} parallel threads"
+      WaybackArchiver.logger.info "Total urls to be sent: #{urls.length}"
 
-      ProcessQueue.process(urls, threads_count: concurrency) { |url| post_url(url) }
+      pool = Concurrent::FixedThreadPool.new(concurrency)
+      urls.each do |url|
+        pool.post { Archive.post_url(url) }
+      end
 
-      puts "#{urls.length} URLs sent to Internet archive"
+      WaybackArchiver.logger.info "#{urls.length} URLs sent to Internet archive"
       urls
     end
 
+    # Send URLs to Wayback Machine by crawling the site.
+    # @return [Array] with URLs sent to the Wayback Machine.
+    # @param [String] source for URL to crawl.
+    # @param [Integer] concurrency (default is 5).
+    # @example Crawl example.com and send all URLs of the same domain
+    #    WaybackArchiver.crawl('example.com')
+    # @example Crawl example.com and send all URLs of the same domain with low concurrency
+    #    WaybackArchiver.crawl('example.com', concurrency: 1)
+    def self.crawl(source, concurrency: DEFAULT_CONCURRENCY)
+      pool = Concurrent::FixedThreadPool.new(concurrency) # X threads
+
+      UrlCollector.crawl(source) do |url|
+        pool.post { Archive.post_url(url) }
+      end
+    end
+
     # Send URL to Wayback Machine.
     # @return [String] the sent URL.
     # @param [String] url to send.
@@ -35,11 +51,11 @@ module WaybackArchiver
     def self.post_url(url)
       request_url  = "#{WAYBACK_BASE_URL}#{url}"
       response     = Request.response(request_url)
-      puts "[#{response.code}, #{response.message}] #{url}"
+      WaybackArchiver.logger.info "[#{response.code}, #{response.message}] #{url}"
       url
     rescue Exception => e
-      puts "Error message:     #{e.message}"
-      puts "Failed to archive: #{url}"
+      WaybackArchiver.logger.error "Error message:     #{e.message}"
+      WaybackArchiver.logger.error "Failed to archive: #{url}"
     end
   end
 end

data/lib/wayback_archiver/null_logger.rb

@@ -0,0 +1,9 @@
+require 'logger'
+
+class NullLogger < Logger
+  def initialize(*args)
+  end
+
+  def add(*args, &block)
+  end
+end

data/lib/wayback_archiver/url_collector.rb

@@ -27,7 +27,7 @@ module WaybackArchiver
         spider.every_html_page do |page|
           page_url = page.url.to_s
           urls << page_url
-          puts "Found: #{page_url}"
+          WaybackArchiver.logger.info "Found: #{page_url}"
           yield(page_url) if block_given?
         end
       end

data/lib/wayback_archiver/version.rb

@@ -1,4 +1,4 @@
 module WaybackArchiver
   # Gem version
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/lib/wayback_archiver.rb

@@ -1,11 +1,13 @@
 require 'uri'
 require 'net/http'
 
+require 'concurrent'
+
+require 'wayback_archiver/null_logger'
 require 'wayback_archiver/version'
 require 'wayback_archiver/url_collector'
 require 'wayback_archiver/archive'
 require 'wayback_archiver/request'
-require 'wayback_archiver/process_queue'
 
 # WaybackArchiver, send URLs to Wayback Machine. By crawling, sitemap, file or single URL.
 module WaybackArchiver
@@ -28,11 +30,31 @@ module WaybackArchiver
   def self.archive(source, type = :crawl)
     case type.to_s
     when 'file'    then Archive.post(UrlCollector.file(source))
-    when 'crawl'   then UrlCollector.crawl(source) { |url| Archive.post_url(url) }
+    when 'crawl'   then crawl(source)
     when 'sitemap' then Archive.post(UrlCollector.sitemap(source))
     when 'url'     then Archive.post_url(Request.resolve_url(source))
     else
       raise ArgumentError, "Unknown type: '#{type}'. Allowed types: sitemap, url, file, crawl"
     end
   end
+
+  # Crawl site for URLs to send to the Wayback Machine.
+  # @return [Array] with URLs sent to the Wayback Machine.
+  # @param [String] source for URL(s).
+  # @param [Integer] concurrency.
+  # @example Crawl example.com and send all URLs of the same domain
+  #    WaybackArchiver.crawl('example.com') # Default concurrency is 5
+  # @example Crawl example.com and send all URLs of the same domain with low concurrency
+  #    WaybackArchiver.crawl('example.com', concurrency: 1)
+  def self.crawl(source, concurrency: Archive::DEFAULT_CONCURRENCY)
+    Archive.crawl(source, concurrency: concurrency)
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.logger
+    @logger ||= NullLogger.new
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-20 00:00:00.000000000 Z
+date: 2017-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -150,7 +164,8 @@ dependencies:
     - - ">"
       - !ruby/object:Gem::Version
         version: '0'
-description: Send URLs to Wayback Machine. By crawling, sitemap, file or single URL.
+description: Send URLs to Wayback Machine (Internet Archive). By crawling, sitemap,
+  file or single URL.
 email:
 - burenstam@gmail.com
 executables:
@@ -161,7 +176,7 @@ files:
 - bin/wayback_archiver
 - lib/wayback_archiver.rb
 - lib/wayback_archiver/archive.rb
-- lib/wayback_archiver/process_queue.rb
+- lib/wayback_archiver/null_logger.rb
 - lib/wayback_archiver/request.rb
 - lib/wayback_archiver/url_collector.rb
 - lib/wayback_archiver/version.rb
@@ -177,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -185,8 +200,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: Send URLs to Wayback Machine
+summary: Send URLs to Wayback Machine (Internet Archive)
 test_files: []

data/lib/wayback_archiver/process_queue.rb

@@ -1,28 +0,0 @@
-require 'thread'
-
-module WaybackArchiver
-  class ProcessQueue
-    # Process enumerable data in parallel.
-    # @return [Array] of URLs defined found during crawl.
-    # @param [Object] Enumberable object
-    # @example Print list of names in parallel
-    #    ProcessQueue.process(%w(jacob peter eva)) { |v| puts n }
-    # @example Print list of names using 2 threads
-    #    ProcessQueue.process(%w(jacob peter eva), threads_count: 2) { |v| puts n }
-    def self.process(data_array, threads_count: 5)
-      queue = Queue.new
-      data_array.each { |data| queue.push(data) }
-      workers = threads_count.times.map do
-        Thread.new do
-          begin
-            while data = queue.pop(true)
-              yield(data)
-            end
-          rescue ThreadError
-          end
-        end
-      end
-      workers.map(&:join)
-    end
-  end
-end