wayback_archiver 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ddbffea2e55297390c66201d287b85fb6336d864
-  data.tar.gz: b419745edba1f8dcf9d6e83ce5b74cd70c9abd0f
+  metadata.gz: b64e289cd025ebddc9af5472a3c5038af4e98535
+  data.tar.gz: 903a6e3f3bfb2f6ab81db471b8633b27f5f8c419
 SHA512:
-  metadata.gz: 5bb23d2bab242cc55d1a9e851e5fd719431371f2149a2640ce34ede4be817f881ace982d39cff04691d435b140bbc54419bf90affb28d0621261cb9ee7d34a69
-  data.tar.gz: 91c6651a5cbeb1333a9f24ab3596ee0b284c54e4aa7c375b158bf6c3bbb54892a6c78c930a1942a3dcc9f4e0ab937c60a08e3f60a8339f0e9342cea6f0959c5f
+  metadata.gz: cc1853305301920afbcdd7a79f592cd6863f27fdb57c22d634424346526441d8ddd788bee8b7e23f938f334fbbd7a2d162b397041ddc6fe0f1193d816d4fac59
+  data.tar.gz: e64370ece062319402c773464493fb767148712e115ae3ce5aebf3b66361dae1f94177a9d50b6aeaf0770d8ddcb86d3e7b779c170bfc6294633fab4283e711c4

data/lib/wayback_archiver/archive.rb

@@ -1,31 +1,27 @@
 module WaybackArchiver
   class Archive
-    MAX_THREAD_COUNT = 8
+    WAYBACK_BASE_URL = 'https://web.archive.org/save/'
+    MAX_THREAD_COUNT = 10
 
-    def self.post(all_urls)
+    def self.post(urls)
       puts "Request are sent with up to #{MAX_THREAD_COUNT} parallel threads"
-  
-      puts "Total urls to be sent: #{all_urls.length}"
-      threads    = []
-      group_size = (all_urls.length / MAX_THREAD_COUNT) + 1
-      all_urls.each_slice(group_size).to_a.each do |urls|
-        threads << Thread.new do
-          urls.each_with_index do |url, index|
-            resolved_url = Request.resolve_url(url)
-            request_url  = "#{BASE_URL}#{resolved_url}"
-            begin
-              res = Request.get_response(request_url)
-              puts "[#{res.code}, #{res.message}] #{resolved_url}"
-            rescue Exception => e
-              puts "Error message: #{e.message}"
-              puts "Failed to archive: #{resolved_url}"
-            end
-          end
-        end
-      end
-      threads.each(&:join)
-      puts "#{all_urls.length} URLs sent to Internet archive"
-      all_urls
+      puts "Total urls to be sent: #{urls.length}"
+      group_size = (urls.length / MAX_THREAD_COUNT) + 1
+      urls.each_slice(group_size).to_a.map do |archive_urls|
+        Thread.new { archive_urls.each { |url| post_url(url) } }
+      end.each(&:join)
+      puts "#{urls.length} URLs sent to Internet archive"
+      urls
+    end
+
+    def self.post_url(archive_url)
+      resolved_url = Request.resolve_url(archive_url)
+      request_url  = "#{WAYBACK_BASE_URL}#{resolved_url}"
+      response     = Request.get_response(request_url)
+      puts "[#{response.code}, #{response.message}] #{resolved_url}"
+    rescue Exception => e
+      puts "Error message:     #{e.message}"
+      puts "Failed to archive: #{resolved_url}"
     end
   end
 end

data/lib/wayback_archiver/collector.rb

@@ -2,12 +2,9 @@ module WaybackArchiver
   class Collector
     class << self
       def urls_from_sitemap(url)
-        urls     = []
-        xml_data = Request.get_response(Request.resolve_url(url)).body
-        document = REXML::Document.new(xml_data)
-
-        document.elements.each('urlset/url/loc') { |element| urls << element.text }
-        urls
+        resolved = Request.resolve_url(url)
+        sitemap  = Request.get_page(resolved)
+        sitemap.css('loc').map { |element| element.text }
       end
 
       def urls_from_crawl(url)

data/lib/wayback_archiver/crawl_url.rb

@@ -4,7 +4,8 @@ module WaybackArchiver
 
     def initialize(base_url)
       @resolved_base_url = Request.resolve_url(base_url)
-      @base_hostname     = URI.parse(@resolved_base_url).host
+      @base_hostname     = URI.parse(@resolved_base_url).hostname
+      @resolved_base_url.prepend('http://') unless @resolved_base_url.start_with?('http')
     end
 
     def absolute_url_from(raw_url, get_url)
@@ -12,7 +13,7 @@ module WaybackArchiver
       parsed_url = URI.parse(raw_url) rescue URI.parse('')  
       if parsed_url.relative?
         url_from_relative(raw_url, get_url)
-      elsif base_hostname.eql?(parsed_url.hostname)
+      elsif same_domain?(raw_url, @resolved_base_url)
         raw_url
       else
         nil
@@ -65,5 +66,9 @@ module WaybackArchiver
       dont_end.each     { |pattern| return false if href.end_with?(pattern) }
       true
     end
+
+    def same_domain?(first, second)
+      first.include?(second)
+    end
   end
 end

data/lib/wayback_archiver/crawler.rb

@@ -1,19 +1,19 @@
 require 'set'
-require 'nokogiri'   
+require 'nokogiri'
 
 module WaybackArchiver
   class Crawler
     CRAWLER_INFO_LINK = 'https://rubygems.org/gems/wayback_archiver'
     HEADERS_HASH      = {
-      'User-Agent' => "WaybackArchiver/#{VERSION} (+#{CRAWLER_INFO_LINK})"
+      'User-Agent' => "WaybackArchiver/#{WaybackArchiver::VERSION} (+#{CRAWLER_INFO_LINK})"
     }
 
-    def initialize(url, resolve: false)
+    def initialize(url, resolve = false)
       base_url     = Request.resolve_url(url)
       @options     = { resolve: resolve }
       @crawl_url   = CrawlUrl.new(base_url)
       @fetch_queue = Set.new
-      @procesed    = Set.new
+      @processed   = Set.new
       @fetch_queue << @crawl_url.resolved_base_url
     end
 
@@ -21,14 +21,14 @@ module WaybackArchiver
       new(base_url).collect_urls
     end
 
-    def collect_urls        
+    def collect_urls
       until @fetch_queue.empty?
         url = @fetch_queue.first
         @fetch_queue.delete(@fetch_queue.first)
         page_links(url)
       end
-      puts "Crawling finished, #{@procesed.length} links found"
-      @procesed.to_a
+      puts "Crawling finished, #{@processed.length} links found"
+      @processed.to_a
     rescue Interrupt, IRB::Abort
       puts 'Crawl interrupted.'
       @fetch_queue.to_a
@@ -38,21 +38,17 @@ module WaybackArchiver
 
     def page_links(get_url)
       puts "Queue length: #{@fetch_queue.length}, Parsing: #{get_url}"
-      link_elements = get_page(get_url).css('a') rescue []
-      @procesed << get_url
+      link_elements = Request.get_page(get_url).css('a') rescue []
+      @processed << get_url
       link_elements.each do |page_link|
         absolute_url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
         if absolute_url
           resolved_url = resolve(absolute_url)
-          @fetch_queue << resolved_url if !@procesed.include?(resolved_url)
+          @fetch_queue << resolved_url if !@processed.include?(resolved_url)
         end
       end
     end
 
-    def get_page(url)
-      Nokogiri::HTML(Request.get_response(url).body)
-    end
-
     def resolve(url)
       @options[:resolve] ? Request.resolve_url(url) : url
     end

data/lib/wayback_archiver/request.rb

@@ -5,19 +5,31 @@ module WaybackArchiver
     INFO_LINK  = 'https://rubygems.org/gems/wayback_archiver'
     USER_AGENT = "WaybackArchiver/#{VERSION} (+#{INFO_LINK})"
 
-    def self.get_response(url, resolve: false)
-      resolved_url = resolve ? resolve_url(url) : url
-      uri          = URI.parse(resolved_url)
-      http         = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true if resolved_url.include?('https://')
+    class << self
+      def get_page(url, document_type = :html)
+        response = Request.get_response(url).body
+        case document_type
+        when :xml
+          Nokogiri::XML(response)
+        else
+          Nokogiri::HTML(response)
+        end
+      end
 
-      request = Net::HTTP::Get.new(uri.request_uri)
-      request['User-Agent'] = USER_AGENT
-      http.request(request)
-    end
+      def get_response(url, resolve = false)
+        resolved_url = resolve ? resolve_url(url) : url
+        uri          = URI.parse(resolved_url)
+        http         = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true if resolved_url.include?('https://')
+
+        request = Net::HTTP::Get.new(uri.request_uri)
+        request['User-Agent'] = USER_AGENT
+        http.request(request)
+      end
 
-    def self.resolve_url(url)
-      UrlResolver.resolve(url)
+      def resolve_url(url)
+        UrlResolver.resolve(url)
+      end
     end
   end
 end

data/lib/wayback_archiver/version.rb

@@ -1,3 +1,3 @@
 module WaybackArchiver
-  VERSION = '0.0.7'
+  VERSION = '0.0.8'
 end

data/lib/wayback_archiver.rb

@@ -1,6 +1,5 @@
 require 'uri'
 require 'net/http'
-require 'rexml/document'
 
 require 'wayback_archiver/collector'
 require 'wayback_archiver/archive'
@@ -9,18 +8,12 @@ require 'wayback_archiver/crawler'
 require 'wayback_archiver/crawl_url'
 
 module WaybackArchiver
-  BASE_URL = 'https://web.archive.org/save/'
-
-  def self.archive(source, from = :sitemap)
+  def self.archive(source, from = :crawl)
     urls = case from.to_s
-    when 'sitemap'
-      Collector.urls_from_sitemap("#{source}/sitemap.xml")
-    when 'url'
-      [Request.resolve_url(source)]
-    when 'file'
-      Collector.urls_from_file(source)
-    when 'crawl', 'crawler'
-      Collector.urls_from_crawl(source)
+    when 'sitemap' then Collector.urls_from_sitemap("#{source}/sitemap.xml")
+    when 'url'     then [Request.resolve_url(source)]
+    when 'file'    then Collector.urls_from_file(source)
+    when 'crawl'   then Collector.urls_from_crawl(source)
     else
       raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemap, url, file, crawl"
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-20 00:00:00.000000000 Z
+date: 2014-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri