wayback_archiver 1.3.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4dba94d5ac29b57a1df0a6e98f5d9ea29c1905b9ffdef7ee9bf9c344448362fe
-  data.tar.gz: 79af5b2af8660dd5f0cafae521e7b73be36e528da7c8ce2f62502f76b434959b
+  metadata.gz: 405a39b07682a9c07ea9cf4689af2f110db21515c8ee192a90092c021fa8cd7b
+  data.tar.gz: 61c8500dc285c5e0975f95bd6ebc86b42bffed5daec93899f90a3ac5782046a0
 SHA512:
-  metadata.gz: d02a18809ebdc880bdcdb9a7d0d522da445c2a2329959080c897d1f7033f6a687c268a626abfa05a45791c5e1404a4e3e8025ea172494bca0b430033e4fc20b0
-  data.tar.gz: 316e8fe4e922b0aeb5047191fa6ddd8e242fa7bfd1ce8eb2b2a2e08d980999dd895c071f82f949a2690c4259bc5b3de00049e691b540040db79d002ffdf8a4f0
+  metadata.gz: cb74bad72fe7a33f8d45ee44afc3beb45424281ad05ca778222f1497eb3e92f4ae4cdde60669bd544436fc64dbd2b7acfdafa403e0757bcbeaf4df6aed9695cc
+  data.tar.gz: 0224a4bee7755dd25d791a2aa83b91960df395ffec1b3db687b73f32541b92ad9a726949926fb744062415903354eea3e4672e7235855d17f362da94fbcb3302

data/bin/wayback_archiver

@@ -10,6 +10,7 @@ log = STDOUT
 log_level = Logger::INFO
 concurrency = WaybackArchiver.concurrency
 limit = WaybackArchiver.max_limit
+hosts = []
 
 optparse = OptionParser.new do |parser|
   parser.banner = 'Usage: wayback_archiver [<url>] [options]'
@@ -30,7 +31,11 @@ optparse = OptionParser.new do |parser|
     strategy = 'urls'
   end
 
-  parser.on('--concurrency=5', Integer, 'Concurrency') do |value|
+  parser.on('--hosts=[example.com]', Array, 'Only spider links on certain hosts') do |value|
+    hosts = value.map { |v| Regexp.new(v) } if value
+  end
+
+  parser.on('--concurrency=1', Integer, 'Concurrency') do |value|
     concurrency = value
   end
 
@@ -81,6 +86,7 @@ strategy ||= 'auto'
 urls.each do |url|
   WaybackArchiver.archive(
     url,
+    hosts: hosts,
     strategy: strategy,
     concurrency: concurrency,
     limit: limit

data/lib/robots.rb

@@ -0,0 +1,162 @@
+#
+#  Copyright (c) 2008 Kyle Maxwell, contributors
+#
+#  Permission is hereby granted, free of charge, to any person
+#  obtaining a copy of this software and associated documentation
+#  files (the "Software"), to deal in the Software without
+#  restriction, including without limitation the rights to use,
+#  copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the
+#  Software is furnished to do so, subject to the following
+#  conditions:
+#
+#  The above copyright notice and this permission notice shall be
+#  included in all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#  OTHER DEALINGS IN THE SOFTWARE.
+#
+
+require "open-uri"
+require "uri"
+require "rubygems"
+require "timeout"
+
+class Robots
+  
+  DEFAULT_TIMEOUT = 3
+  
+  class ParsedRobots
+    
+    def initialize(uri, user_agent)
+      @last_accessed = Time.at(1)
+      
+      io = Robots.get_robots_txt(uri, user_agent)
+      
+      if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
+        io = StringIO.new("User-agent: *\nAllow: /\n")
+      end
+
+      @other = {}
+      @disallows = {}
+      @allows = {}
+      @delays = {} # added delays to make it work
+      agent = /.*/
+      io.each do |line|
+        next if line =~ /^\s*(#.*|$)/
+        arr = line.split(":")
+        key = arr.shift
+        value = arr.join(":").strip
+        value.strip!
+        case key
+        when "User-agent"
+          agent = to_regex(value)
+        when "Allow"
+          @allows[agent] ||= []
+          @allows[agent] << to_regex(value)
+        when "Disallow"
+          @disallows[agent] ||= []
+          @disallows[agent] << to_regex(value)
+        when "Crawl-delay"
+          @delays[agent] = value.to_i
+        else
+          @other[key] ||= []
+          @other[key] << value
+        end
+      end
+      
+      @parsed = true
+    end
+    
+    def allowed?(uri, user_agent)
+      return true unless @parsed
+      allowed = true
+      path = uri.request_uri
+      
+      @disallows.each do |key, value|
+        if user_agent =~ key
+          value.each do |rule|
+            if path =~ rule
+              allowed = false
+            end
+          end
+        end
+      end
+      
+      @allows.each do |key, value|
+        unless allowed      
+          if user_agent =~ key
+            value.each do |rule|
+              if path =~ rule
+                allowed = true
+              end
+            end
+          end
+        end
+      end
+      
+      if allowed && @delays[user_agent]
+        sleep @delays[user_agent] - (Time.now - @last_accessed)
+        @last_accessed = Time.now
+      end
+      
+      return allowed
+    end
+    
+    def other_values
+      @other
+    end
+    
+  protected
+    
+    def to_regex(pattern)
+      return /should-not-match-anything-123456789/ if pattern.strip.empty?
+      pattern = Regexp.escape(pattern)
+      pattern.gsub!(Regexp.escape("*"), ".*")
+      Regexp.compile("^#{pattern}")
+    end
+  end
+  
+  def self.get_robots_txt(uri, user_agent)
+    begin
+      Timeout::timeout(Robots.timeout) do
+        io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
+      end 
+    rescue Timeout::Error
+      STDERR.puts "robots.txt request timed out"
+    end
+  end
+  
+  def self.timeout=(t)
+    @timeout = t
+  end
+  
+  def self.timeout
+    @timeout || DEFAULT_TIMEOUT
+  end
+  
+  def initialize(user_agent)
+    @user_agent = user_agent
+    @parsed = {}
+  end
+  
+  def allowed?(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri, @user_agent)
+    @parsed[host].allowed?(uri, @user_agent)
+  end
+  
+  def other_values(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri, @user_agent)
+    @parsed[host].other_values
+  end
+end

data/lib/wayback_archiver/adapters/wayback_machine.rb

@@ -13,7 +13,7 @@ module WaybackArchiver
     # @example Archive example.com, with default options
     #    WaybackMachine.call('http://example.com')
     def self.call(url)
-      request_url  = "#{BASE_URL}#{url}"
+      request_url  = "#{BASE_URL}#{url&.strip}"
       response = Request.get(request_url, follow_redirects: false)
       WaybackArchiver.logger.info "Posted [#{response.code}, #{response.message}] #{url}"
       ArchiveResult.new(

data/lib/wayback_archiver/archive.rb

@@ -9,7 +9,7 @@ module WaybackArchiver
     # Send URLs to Wayback Machine.
     # @return [Array<ArchiveResult>] with sent URLs.
     # @param [Array<String>] urls to send to the Wayback Machine.
-    # @param concurrency [Integer] the default is 5
+    # @param concurrency [Integer] the default is 1
     # @yield [archive_result] If a block is given, each result will be yielded
     # @yieldparam [ArchiveResult] archive_result
     # @example Archive urls, asynchronously
@@ -54,7 +54,8 @@ module WaybackArchiver
     # Send URLs to Wayback Machine by crawling the site.
     # @return [Array<ArchiveResult>] with URLs sent to the Wayback Machine.
     # @param [String] source for URL to crawl.
-    # @param concurrency [Integer] the default is 5
+    # @param concurrency [Integer] the default is 1
+    # @param [Array<String, Regexp>] hosts to crawl
     # @yield [archive_result] If a block is given, each result will be yielded
     # @yieldparam [ArchiveResult] archive_result
     # @example Crawl example.com and send all URLs of the same domain
@@ -66,13 +67,21 @@ module WaybackArchiver
     #    Archiver.crawl('example.com', concurrency: 1)
     # @example Stop after archiving 100 links
     #    Archiver.crawl('example.com', limit: 100)
-    def self.crawl(source, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit)
+    # @example Crawl multiple hosts
+    #    URLCollector.crawl(
+    #      'http://example.com',
+    #      hosts: [
+    #        'example.com',
+    #        /host[\d]+\.example\.com/
+    #      ]
+    #    )
+    def self.crawl(source, hosts: [], concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit)
       WaybackArchiver.logger.info "Request are sent with up to #{concurrency} parallel threads"
 
       posted_urls = Concurrent::Array.new
       pool = ThreadPool.build(concurrency)
 
-      found_urls = URLCollector.crawl(source, limit: limit) do |url|
+      found_urls = URLCollector.crawl(source, hosts: hosts, limit: limit) do |url|
         pool.post do
           result = post_url(url)
           yield(result) if block_given?

data/lib/wayback_archiver/sitemap.rb

@@ -1,3 +1,4 @@
+require 'uri'
 require 'rexml/document'
 
 module WaybackArchiver
@@ -5,8 +6,9 @@ module WaybackArchiver
   class Sitemap
     attr_reader :document
 
-    def initialize(xml, strict: false)
-      @document = REXML::Document.new(xml)
+    def initialize(xml_or_string, strict: false)
+      @contents = xml_or_string
+      @document = REXML::Document.new(xml_or_string)
     rescue REXML::ParseException => _e
       raise if strict
 
@@ -65,9 +67,20 @@ module WaybackArchiver
 
     private
 
+    def valid_url?(url)
+      uri = URI.parse(url)
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+
     # Extract URLs from Sitemap
     def extract_urls(node_name)
-      return document.to_s.each_line.map(&:strip) if plain_document?
+      if plain_document?
+        return @contents.to_s
+          .each_line.map(&:strip)
+          .select(&method(:valid_url?))
+      end
 
       urls = []
       document.root.elements.each("#{node_name}/loc") do |element|

data/lib/wayback_archiver/sitemapper.rb

@@ -79,7 +79,7 @@ module WaybackArchiver
           urls(url: sitemap_url, visited: visited)
         end
       else
-        sitemap.urls
+        sitemap.urls.map { |url| url&.strip }
       end
     rescue Request::Error => e
       WaybackArchiver.logger.error "Error raised when requesting #{url}, #{e.class}, #{e.message}"

data/lib/wayback_archiver/url_collector.rb

@@ -19,22 +19,32 @@ module WaybackArchiver
     # Retrieve URLs by crawling.
     # @return [Array<String>] of URLs defined found during crawl.
     # @param [String] url domain to crawl URLs from.
+    # @param [Array<String, Regexp>] hosts to crawl.
     # @example Crawl URLs defined on example.com
     #    URLCollector.crawl('http://example.com')
     # @example Crawl URLs defined on example.com and limit the number of visited pages to 100
     #    URLCollector.crawl('http://example.com', limit: 100)
     # @example Crawl URLs defined on example.com and explicitly set no upper limit on the number of visited pages to 100
     #    URLCollector.crawl('http://example.com', limit: -1)
-    def self.crawl(url, limit: WaybackArchiver.max_limit)
+    # @example Crawl multiple hosts
+    #    URLCollector.crawl(
+    #      'http://example.com',
+    #      hosts: [
+    #        'example.com',
+    #        /host[\d]+\.example\.com/
+    #      ]
+    #    )
+    def self.crawl(url, hosts: [], limit: WaybackArchiver.max_limit)
       urls = []
       start_at_url = Request.build_uri(url).to_s
       options = {
-        robots: true,
+        robots: WaybackArchiver.respect_robots_txt,
+        hosts: hosts,
         user_agent: WaybackArchiver.user_agent
       }
       options[:limit] = limit unless limit == -1
 
-      Spidr.site(start_at_url, options) do |spider|
+      Spidr.site(start_at_url, **options) do |spider|
         spider.every_page do |page|
           page_url = page.url.to_s
           urls << page_url

data/lib/wayback_archiver/version.rb

@@ -1,4 +1,4 @@
 module WaybackArchiver
   # Gem version
-  VERSION = '1.3.0'.freeze
+  VERSION = '1.5.0'.freeze
 end

data/lib/wayback_archiver.rb

@@ -11,9 +11,11 @@ module WaybackArchiver
   INFO_LINK  = 'https://rubygems.org/gems/wayback_archiver'.freeze
   # WaybackArchiver User-Agent
   USER_AGENT = "WaybackArchiver/#{WaybackArchiver::VERSION} (+#{INFO_LINK})".freeze
+  # Default for whether to respect robots txt files
+  DEFAULT_RESPECT_ROBOTS_TXT = false
 
   # Default concurrency for archiving URLs
-  DEFAULT_CONCURRENCY = 5
+  DEFAULT_CONCURRENCY = 1
 
   # Maxmium number of links posted (-1 is no limit)
   DEFAULT_MAX_LIMIT = -1
@@ -22,6 +24,7 @@ module WaybackArchiver
   # @return [Array<ArchiveResult>] of URLs sent to the Wayback Machine.
   # @param [String/Array<String>] source for URL(s).
   # @param [String/Symbol] strategy of source. Supported strategies: crawl, sitemap, url, urls, auto.
+  # @param [Array<String, Regexp>] hosts to crawl.
   # @example Crawl example.com and send all URLs of the same domain
   #    WaybackArchiver.archive('example.com') # Default strategy is :auto
   #    WaybackArchiver.archive('example.com', strategy: :auto)
@@ -43,11 +46,19 @@ module WaybackArchiver
   #    WaybackArchiver.archive('example.com', strategy: :url, concurrency: 10)
   #    WaybackArchiver.archive('example.com', strategy: :url, limit: 100) # send max 100 URLs
   #    WaybackArchiver.archive('example.com', :url)
-  def self.archive(source, legacy_strategy = nil, strategy: :auto, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
+  # @example Crawl multiple hosts
+  #    WaybackArchiver.archive(
+  #      'http://example.com',
+  #      hosts: [
+  #        'example.com',
+  #        /host[\d]+\.example\.com/
+  #      ]
+  #    )
+  def self.archive(source, legacy_strategy = nil, strategy: :auto, hosts: [], concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
     strategy = legacy_strategy || strategy
 
     case strategy.to_s
-    when 'crawl'   then crawl(source, concurrency: concurrency, limit: limit, &block)
+    when 'crawl'   then crawl(source, concurrency: concurrency, limit: limit, hosts: hosts, &block)
     when 'auto'    then auto(source, concurrency: concurrency, limit: limit, &block)
     when 'sitemap' then sitemap(source, concurrency: concurrency, limit: limit, &block)
     when 'urls'    then urls(source, concurrency: concurrency, limit: limit, &block)
@@ -63,7 +74,7 @@ module WaybackArchiver
   # @param [String] source (must be a valid URL).
   # @param concurrency [Integer]
   # @example Auto archive example.com
-  #    WaybackArchiver.auto('example.com') # Default concurrency is 5
+  #    WaybackArchiver.auto('example.com') # Default concurrency is 1
   # @example Auto archive example.com with low concurrency
   #    WaybackArchiver.auto('example.com', concurrency: 1)
   # @example Auto archive example.com and archive max 100 URLs
@@ -79,16 +90,25 @@ module WaybackArchiver
   # Crawl site for URLs to send to the Wayback Machine.
   # @return [Array<ArchiveResult>] of URLs sent to the Wayback Machine.
   # @param [String] url to start crawling from.
+  # @param [Array<String, Regexp>] hosts to crawl
   # @param concurrency [Integer]
   # @example Crawl example.com and send all URLs of the same domain
-  #    WaybackArchiver.crawl('example.com') # Default concurrency is 5
+  #    WaybackArchiver.crawl('example.com') # Default concurrency is 1
   # @example Crawl example.com and send all URLs of the same domain with low concurrency
   #    WaybackArchiver.crawl('example.com', concurrency: 1)
   # @example Crawl example.com and archive max 100 URLs
   #    WaybackArchiver.crawl('example.com', limit: 100)
-  def self.crawl(url, concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
+  # @example Crawl multiple hosts
+  #    URLCollector.crawl(
+  #      'http://example.com',
+  #      hosts: [
+  #        'example.com',
+  #        /host[\d]+\.example\.com/
+  #      ]
+  #    )
+  def self.crawl(url, hosts: [], concurrency: WaybackArchiver.concurrency, limit: WaybackArchiver.max_limit, &block)
     WaybackArchiver.logger.info "Crawling #{url}"
-    Archive.crawl(url, concurrency: concurrency, limit: limit, &block)
+    Archive.crawl(url, hosts: hosts, concurrency: concurrency, limit: limit, &block)
   end
 
   # Get URLs from sitemap and send found URLs to the Wayback Machine.
@@ -96,7 +116,7 @@ module WaybackArchiver
   # @param [String] url to the sitemap.
   # @param concurrency [Integer]
   # @example Get example.com sitemap and archive all found URLs
-  #    WaybackArchiver.sitemap('example.com/sitemap.xml') # Default concurrency is 5
+  #    WaybackArchiver.sitemap('example.com/sitemap.xml') # Default concurrency is 1
   # @example Get example.com sitemap and archive all found URLs with low concurrency
   #    WaybackArchiver.sitemap('example.com/sitemap.xml', concurrency: 1)
   # @example Get example.com sitemap archive max 100 URLs
@@ -155,6 +175,19 @@ module WaybackArchiver
     @user_agent ||= USER_AGENT
   end
 
+  # Sets the default respect_robots_txt
+  # @return [Boolean] the desired default for respect_robots_txt
+  # @param [Boolean] respect_robots_txt the desired default
+  def self.respect_robots_txt=(respect_robots_txt)
+    @respect_robots_txt = respect_robots_txt
+  end
+
+  # Returns the default respect_robots_txt
+  # @return [Boolean] the configured or the default respect_robots_txt
+  def self.respect_robots_txt
+    @respect_robots_txt ||= DEFAULT_RESPECT_ROBOTS_TXT
+  end
+
   # Sets the default concurrency
   # @return [Integer] the desired default concurrency
   # @param [Integer] concurrency the desired default concurrency

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Jacob Burenstam
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-24 00:00:00.000000000 Z
+date: 2024-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr
@@ -16,70 +16,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.7.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.7.1
 - !ruby/object:Gem::Dependency
-  name: robots
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: rexml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 3.3.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 3.3.9
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.3'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.3'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -168,16 +168,16 @@ dependencies:
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.1.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.1.3
 description: Post URLs to Wayback Machine (Internet Archive), using a crawler, from
   Sitemap(s) or a list of URLs.
 email:
@@ -188,6 +188,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/wayback_archiver
+- lib/robots.rb
 - lib/wayback_archiver.rb
 - lib/wayback_archiver/adapters/wayback_machine.rb
 - lib/wayback_archiver/archive.rb
@@ -205,7 +206,7 @@ homepage: https://github.com/buren/wayback_archiver
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -220,9 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.5.3
+signing_key:
 specification_version: 4
 summary: Post URLs to Wayback Machine (Internet Archive)
 test_files: []