wayback_archiver 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5f39f42fe6d5a4f6fbded5ab460cd91a2c9411af28645a41b987bf01a31ea14
-  data.tar.gz: 8cc1f5dbdc7d55fb9a1ec358c354fd8db3bacd856a824e9090005445a966de39
+  metadata.gz: 405a39b07682a9c07ea9cf4689af2f110db21515c8ee192a90092c021fa8cd7b
+  data.tar.gz: 61c8500dc285c5e0975f95bd6ebc86b42bffed5daec93899f90a3ac5782046a0
 SHA512:
-  metadata.gz: 6e3edf351b7cda562d39120df2dff564a8248c41ecedae54deed2017c11cbc6c56311507a7dda87ce350b17ac9bdf3d265523dde3a79de751426d206a2f51741
-  data.tar.gz: 020cb49d6dfc204de93a035853a26448b20901dc962d772554c4cb9074ff14682c570afffe3c2cbc05984c2298a7a05ffe662f8aaccde80e510b97450dd270cc
+  metadata.gz: cb74bad72fe7a33f8d45ee44afc3beb45424281ad05ca778222f1497eb3e92f4ae4cdde60669bd544436fc64dbd2b7acfdafa403e0757bcbeaf4df6aed9695cc
+  data.tar.gz: 0224a4bee7755dd25d791a2aa83b91960df395ffec1b3db687b73f32541b92ad9a726949926fb744062415903354eea3e4672e7235855d17f362da94fbcb3302

data/lib/robots.rb

@@ -0,0 +1,162 @@
+#
+#  Copyright (c) 2008 Kyle Maxwell, contributors
+#
+#  Permission is hereby granted, free of charge, to any person
+#  obtaining a copy of this software and associated documentation
+#  files (the "Software"), to deal in the Software without
+#  restriction, including without limitation the rights to use,
+#  copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the
+#  Software is furnished to do so, subject to the following
+#  conditions:
+#
+#  The above copyright notice and this permission notice shall be
+#  included in all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#  OTHER DEALINGS IN THE SOFTWARE.
+#
+
+require "open-uri"
+require "uri"
+require "rubygems"
+require "timeout"
+
+class Robots
+  
+  DEFAULT_TIMEOUT = 3
+  
+  class ParsedRobots
+    
+    def initialize(uri, user_agent)
+      @last_accessed = Time.at(1)
+      
+      io = Robots.get_robots_txt(uri, user_agent)
+      
+      if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
+        io = StringIO.new("User-agent: *\nAllow: /\n")
+      end
+
+      @other = {}
+      @disallows = {}
+      @allows = {}
+      @delays = {} # added delays to make it work
+      agent = /.*/
+      io.each do |line|
+        next if line =~ /^\s*(#.*|$)/
+        arr = line.split(":")
+        key = arr.shift
+        value = arr.join(":").strip
+        value.strip!
+        case key
+        when "User-agent"
+          agent = to_regex(value)
+        when "Allow"
+          @allows[agent] ||= []
+          @allows[agent] << to_regex(value)
+        when "Disallow"
+          @disallows[agent] ||= []
+          @disallows[agent] << to_regex(value)
+        when "Crawl-delay"
+          @delays[agent] = value.to_i
+        else
+          @other[key] ||= []
+          @other[key] << value
+        end
+      end
+      
+      @parsed = true
+    end
+    
+    def allowed?(uri, user_agent)
+      return true unless @parsed
+      allowed = true
+      path = uri.request_uri
+      
+      @disallows.each do |key, value|
+        if user_agent =~ key
+          value.each do |rule|
+            if path =~ rule
+              allowed = false
+            end
+          end
+        end
+      end
+      
+      @allows.each do |key, value|
+        unless allowed      
+          if user_agent =~ key
+            value.each do |rule|
+              if path =~ rule
+                allowed = true
+              end
+            end
+          end
+        end
+      end
+      
+      if allowed && @delays[user_agent]
+        sleep @delays[user_agent] - (Time.now - @last_accessed)
+        @last_accessed = Time.now
+      end
+      
+      return allowed
+    end
+    
+    def other_values
+      @other
+    end
+    
+  protected
+    
+    def to_regex(pattern)
+      return /should-not-match-anything-123456789/ if pattern.strip.empty?
+      pattern = Regexp.escape(pattern)
+      pattern.gsub!(Regexp.escape("*"), ".*")
+      Regexp.compile("^#{pattern}")
+    end
+  end
+  
+  def self.get_robots_txt(uri, user_agent)
+    begin
+      Timeout::timeout(Robots.timeout) do
+        io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
+      end 
+    rescue Timeout::Error
+      STDERR.puts "robots.txt request timed out"
+    end
+  end
+  
+  def self.timeout=(t)
+    @timeout = t
+  end
+  
+  def self.timeout
+    @timeout || DEFAULT_TIMEOUT
+  end
+  
+  def initialize(user_agent)
+    @user_agent = user_agent
+    @parsed = {}
+  end
+  
+  def allowed?(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri, @user_agent)
+    @parsed[host].allowed?(uri, @user_agent)
+  end
+  
+  def other_values(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri, @user_agent)
+    @parsed[host].other_values
+  end
+end

data/lib/wayback_archiver/adapters/wayback_machine.rb

@@ -13,7 +13,7 @@ module WaybackArchiver
     # @example Archive example.com, with default options
     #    WaybackMachine.call('http://example.com')
     def self.call(url)
-      request_url  = "#{BASE_URL}#{url}"
+      request_url  = "#{BASE_URL}#{url&.strip}"
       response = Request.get(request_url, follow_redirects: false)
       WaybackArchiver.logger.info "Posted [#{response.code}, #{response.message}] #{url}"
       ArchiveResult.new(

data/lib/wayback_archiver/sitemap.rb

@@ -1,3 +1,4 @@
+require 'uri'
 require 'rexml/document'
 
 module WaybackArchiver
@@ -5,8 +6,9 @@ module WaybackArchiver
   class Sitemap
     attr_reader :document
 
-    def initialize(xml, strict: false)
-      @document = REXML::Document.new(xml)
+    def initialize(xml_or_string, strict: false)
+      @contents = xml_or_string
+      @document = REXML::Document.new(xml_or_string)
     rescue REXML::ParseException => _e
       raise if strict
 
@@ -65,9 +67,20 @@ module WaybackArchiver
 
     private
 
+    def valid_url?(url)
+      uri = URI.parse(url)
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+
     # Extract URLs from Sitemap
     def extract_urls(node_name)
-      return document.to_s.each_line.map(&:strip) if plain_document?
+      if plain_document?
+        return @contents.to_s
+          .each_line.map(&:strip)
+          .select(&method(:valid_url?))
+      end
 
       urls = []
       document.root.elements.each("#{node_name}/loc") do |element|

data/lib/wayback_archiver/sitemapper.rb

@@ -79,7 +79,7 @@ module WaybackArchiver
           urls(url: sitemap_url, visited: visited)
         end
       else
-        sitemap.urls
+        sitemap.urls.map { |url| url&.strip }
       end
     rescue Request::Error => e
       WaybackArchiver.logger.error "Error raised when requesting #{url}, #{e.class}, #{e.message}"

data/lib/wayback_archiver/url_collector.rb

@@ -44,7 +44,7 @@ module WaybackArchiver
       }
       options[:limit] = limit unless limit == -1
 
-      Spidr.site(start_at_url, options) do |spider|
+      Spidr.site(start_at_url, **options) do |spider|
         spider.every_page do |page|
           page_url = page.url.to_s
           urls << page_url

data/lib/wayback_archiver/version.rb

@@ -1,4 +1,4 @@
 module WaybackArchiver
   # Gem version
-  VERSION = '1.4.0'.freeze
+  VERSION = '1.5.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Jacob Burenstam
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-23 00:00:00.000000000 Z
+date: 2024-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr
@@ -16,42 +16,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.7.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.7.1
 - !ruby/object:Gem::Dependency
-  name: robots
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: rexml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 3.3.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 3.3.9
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -168,16 +168,16 @@ dependencies:
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.1.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.1.3
 description: Post URLs to Wayback Machine (Internet Archive), using a crawler, from
   Sitemap(s) or a list of URLs.
 email:
@@ -188,6 +188,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/wayback_archiver
+- lib/robots.rb
 - lib/wayback_archiver.rb
 - lib/wayback_archiver/adapters/wayback_machine.rb
 - lib/wayback_archiver/archive.rb
@@ -205,7 +206,7 @@ homepage: https://github.com/buren/wayback_archiver
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -220,8 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.5.3
+signing_key:
 specification_version: 4
 summary: Post URLs to Wayback Machine (Internet Archive)
 test_files: []