wayback_archiver 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aee3e5d863178ea9283eb2b723e347d17aefa6b1
-  data.tar.gz: 597d40bdfa0d7538477dbbba3f6671a72f22d82a
+  metadata.gz: ddbffea2e55297390c66201d287b85fb6336d864
+  data.tar.gz: b419745edba1f8dcf9d6e83ce5b74cd70c9abd0f
 SHA512:
-  metadata.gz: 37f2b0372499d7999eed48a80647218127de4d240c661260f18011456c281956cdf778aa3bd66852be08caa3f0cfc03fdf8668895644197c31d1889cb464ddbe
-  data.tar.gz: 505e63ed0e5ce661d96acee3e53d2c950f1e550e611ce64951d7011e76642f595f26e1cc6f522581e3253d1317c1496e861ace64a19032418deb7d14c4f4f991
+  metadata.gz: 5bb23d2bab242cc55d1a9e851e5fd719431371f2149a2640ce34ede4be817f881ace982d39cff04691d435b140bbc54419bf90affb28d0621261cb9ee7d34a69
+  data.tar.gz: 91c6651a5cbeb1333a9f24ab3596ee0b284c54e4aa7c375b158bf6c3bbb54892a6c78c930a1942a3dcc9f4e0ab937c60a08e3f60a8339f0e9342cea6f0959c5f

data/lib/wayback_archiver/archive.rb

@@ -3,7 +3,7 @@ module WaybackArchiver
     MAX_THREAD_COUNT = 8
 
     def self.post(all_urls)
-      puts "Request will be sent with max #{MAX_THREAD_COUNT} parallel threads"
+      puts "Request are sent with up to #{MAX_THREAD_COUNT} parallel threads"
   
       puts "Total urls to be sent: #{all_urls.length}"
       threads    = []
@@ -11,18 +11,20 @@ module WaybackArchiver
       all_urls.each_slice(group_size).to_a.each do |urls|
         threads << Thread.new do
           urls.each_with_index do |url, index|
-            request_url = "#{BASE_URL}#{url}"
+            resolved_url = Request.resolve_url(url)
+            request_url  = "#{BASE_URL}#{resolved_url}"
             begin
               res = Request.get_response(request_url)
-              print "#{url}    #{res.code} => #{res.message} \n"
+              puts "[#{res.code}, #{res.message}] #{resolved_url}"
             rescue Exception => e
               puts "Error message: #{e.message}"
-              puts "Failed to archive: #{url}"
+              puts "Failed to archive: #{resolved_url}"
             end
           end
         end
       end
       threads.each(&:join)
+      puts "#{all_urls.length} URLs sent to Internet archive"
       all_urls
     end
   end

data/lib/wayback_archiver/collector.rb

@@ -1,27 +1,27 @@
 module WaybackArchiver
   class Collector
-
     class << self
-
       def urls_from_sitemap(url)
-        urls     = Array.new
-        xml_data = Request.get_response(url).body
+        urls     = []
+        xml_data = Request.get_response(Request.resolve_url(url)).body
         document = REXML::Document.new(xml_data)
 
         document.elements.each('urlset/url/loc') { |element| urls << element.text }
         urls
       end
 
+      def urls_from_crawl(url)
+        Crawler.collect_urls(url)
+      end
+
       def urls_from_file(path)
         raise ArgumentError, "No such file: #{path}" unless File.exist?(path)
-        urls = Array.new
+        urls = []
         text = File.open(path).read
         text.gsub!(/\r\n?/, "\n") # Normalize line endings
         text.each_line { |line| urls << line.gsub(/\n/, '').strip }
         urls.reject(&:empty?)
       end
-
     end
-
   end
 end

data/lib/wayback_archiver/crawl_url.rb

@@ -0,0 +1,69 @@
+module WaybackArchiver
+  class CrawlUrl
+    attr_reader :resolved_base_url, :base_hostname
+
+    def initialize(base_url)
+      @resolved_base_url = Request.resolve_url(base_url)
+      @base_hostname     = URI.parse(@resolved_base_url).host
+    end
+
+    def absolute_url_from(raw_url, get_url)
+      return nil unless eligible_url?(raw_url)
+      parsed_url = URI.parse(raw_url) rescue URI.parse('')  
+      if parsed_url.relative?
+        url_from_relative(raw_url, get_url)
+      elsif base_hostname.eql?(parsed_url.hostname)
+        raw_url
+      else
+        nil
+      end
+    end
+
+    private
+
+    def url_from_relative(url, current_page_url)
+      if url.start_with?('/')
+        "#{without_path_suffix(resolved_base_url)}#{url}"
+      elsif url.start_with?('../')
+        "#{url_from_dotted_url(url, current_page_url)}"
+      else
+        "#{with_path_suffix(resolved_base_url)}#{url}"
+      end
+    end
+
+    def url_from_dotted_url(url, current_page_url)
+      absolute_url = with_path_suffix(current_page_url.dup)
+      found_dots   = without_path_suffix(url).scan('../').length
+      removed_dots = 0
+      max_levels   = 4
+      while found_dots >= removed_dots && max_levels > removed_dots
+        index = absolute_url.rindex('/') or break
+        absolute_url = absolute_url[0..(index - 1)]
+        removed_dots += 1
+      end
+      "#{with_path_suffix(absolute_url)}#{url.gsub('../', '')}"
+    end
+
+    def with_path_suffix(passed_url)
+      url = passed_url.dup
+      url.end_with?('/') ? url : url << '/'
+    end
+
+    def without_path_suffix(passed_url)
+      url = passed_url.dup
+      url.end_with?('/') ? url[0...(url.length - 1)] : url
+    end
+
+    def eligible_url?(href)
+      return false if href.nil? || href.empty?
+      dont_start   = %w(javascript: callto: mailto: tel: skype: facetime: wtai: #)
+      dont_include = %w(/email-protection#)
+      dont_end     = %w(.zip .rar .pdf .exe .dmg .pkg .dpkg .bat)
+
+      dont_start.each   { |pattern| return false if href.start_with?(pattern) }      
+      dont_include.each { |pattern| return false if href.include?(pattern) }
+      dont_end.each     { |pattern| return false if href.end_with?(pattern) }
+      true
+    end
+  end
+end

data/lib/wayback_archiver/crawler.rb

@@ -1,22 +1,27 @@
 require 'set'
 require 'nokogiri'   
-require 'open-uri'
 
 module WaybackArchiver
   class Crawler
-    def initialize(base_url)
-      @base_url    = base_url
-      @hostname    = URI.parse(@base_url).host
+    CRAWLER_INFO_LINK = 'https://rubygems.org/gems/wayback_archiver'
+    HEADERS_HASH      = {
+      'User-Agent' => "WaybackArchiver/#{VERSION} (+#{CRAWLER_INFO_LINK})"
+    }
+
+    def initialize(url, resolve: false)
+      base_url     = Request.resolve_url(url)
+      @options     = { resolve: resolve }
+      @crawl_url   = CrawlUrl.new(base_url)
       @fetch_queue = Set.new
       @procesed    = Set.new
-      @fetch_queue << @base_url
+      @fetch_queue << @crawl_url.resolved_base_url
     end
 
     def self.collect_urls(base_url)
       new(base_url).collect_urls
     end
 
-    def collect_urls
+    def collect_urls        
       until @fetch_queue.empty?
         url = @fetch_queue.first
         @fetch_queue.delete(@fetch_queue.first)
@@ -24,27 +29,32 @@ module WaybackArchiver
       end
       puts "Crawling finished, #{@procesed.length} links found"
       @procesed.to_a
+    rescue Interrupt, IRB::Abort
+      puts 'Crawl interrupted.'
+      @fetch_queue.to_a
     end
 
-    def page_links(url)
-      puts "Queue length: #{@fetch_queue.length}, Parsing: #{url}"
-      link_elements = Nokogiri::HTML(open(url)).css('a') rescue []
-      @procesed << url
-      link_elements.each do |link|
-        href = sanitize_url(link.attr('href'))
-        @fetch_queue << href if href && !@procesed.include?(href)
+    private
+
+    def page_links(get_url)
+      puts "Queue length: #{@fetch_queue.length}, Parsing: #{get_url}"
+      link_elements = get_page(get_url).css('a') rescue []
+      @procesed << get_url
+      link_elements.each do |page_link|
+        absolute_url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
+        if absolute_url
+          resolved_url = resolve(absolute_url)
+          @fetch_queue << resolved_url if !@procesed.include?(resolved_url)
+        end
       end
     end
 
-    def sanitize_url(raw_url)
-      url = URI.parse(raw_url) rescue URI.parse('')  
-      if url.host.nil?
-        sanitized_url  = "#{@base_url}#{url.path}"
-        sanitized_url += "?#{url.query}" unless url.query.nil?
-        sanitized_url
-      else
-        raw_url if raw_url.include?(@base_url) && @hostname.eql?(url.hostname)
-      end
+    def get_page(url)
+      Nokogiri::HTML(Request.get_response(url).body)
+    end
+
+    def resolve(url)
+      @options[:resolve] ? Request.resolve_url(url) : url
     end
   end
 end

data/lib/wayback_archiver/request.rb

@@ -1,16 +1,23 @@
+require 'url_resolver' # TODO: Allow users to use any resolver
+
 module WaybackArchiver
   class Request
+    INFO_LINK  = 'https://rubygems.org/gems/wayback_archiver'
+    USER_AGENT = "WaybackArchiver/#{VERSION} (+#{INFO_LINK})"
 
-    def self.get_response(url)
-      uri = URI.parse(url)
-
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true if url.include?('https://')
+    def self.get_response(url, resolve: false)
+      resolved_url = resolve ? resolve_url(url) : url
+      uri          = URI.parse(resolved_url)
+      http         = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if resolved_url.include?('https://')
 
-      request  = Net::HTTP::Get.new(uri.request_uri)
-      response = http.request(request)
-      response
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request['User-Agent'] = USER_AGENT
+      http.request(request)
     end
 
+    def self.resolve_url(url)
+      UrlResolver.resolve(url)
+    end
   end
 end

data/lib/wayback_archiver/version.rb

@@ -1,3 +1,3 @@
 module WaybackArchiver
-  VERSION = '0.0.6'
+  VERSION = '0.0.7'
 end

data/lib/wayback_archiver.rb

@@ -6,6 +6,7 @@ require 'wayback_archiver/collector'
 require 'wayback_archiver/archive'
 require 'wayback_archiver/request'
 require 'wayback_archiver/crawler'
+require 'wayback_archiver/crawl_url'
 
 module WaybackArchiver
   BASE_URL = 'https://web.archive.org/save/'
@@ -15,15 +16,14 @@ module WaybackArchiver
     when 'sitemap'
       Collector.urls_from_sitemap("#{source}/sitemap.xml")
     when 'url'
-      Array(source)
+      [Request.resolve_url(source)]
     when 'file'
       Collector.urls_from_file(source)
     when 'crawl', 'crawler'
-      Crawler.collect_urls(source)
+      Collector.urls_from_crawl(source)
     else
-      raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemap, url, file, crawler"
+      raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemap, url, file, crawl"
     end
     Archive.post(urls)
   end
-
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Jacob Burenstam
@@ -24,6 +24,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: url_resolver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +66,48 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'Send URLs to Wayback Machine. From: sitemap, file or single URL.'
 email:
 - burenstam@gmail.com
@@ -63,6 +119,7 @@ files:
 - bin/wayback_archiver
 - lib/wayback_archiver/archive.rb
 - lib/wayback_archiver/collector.rb
+- lib/wayback_archiver/crawl_url.rb
 - lib/wayback_archiver/crawler.rb
 - lib/wayback_archiver/request.rb
 - lib/wayback_archiver/version.rb