wayback_archiver 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3df67fcb7faa06087ad72e6c57ff985d9dbe4fe9
-  data.tar.gz: 5c21d0bf3353dd2ce732cef3b59cd3ba807acc8a
+  metadata.gz: fa0d72d6164b6b280db7c683535e6894ba64778d
+  data.tar.gz: 629963ef30283123820418a714ae73160fa20293
 SHA512:
-  metadata.gz: d21d169fb9fa143b777d771a8cdc5ba8c3c4186f6472aa8f2ff2806034becf18e3cc260c7ce0a9b97d1148a2d1781d9ddc0daef08d7abe1d2b13140125513231
-  data.tar.gz: 15a413ab5bd0f51c40640cd0be764142ca0380315f97fc335cf1bf19f4645b327dc0716400c6c10af7a79b8ceb898b6e0f89a21114865af7f84da1e2bf3d383c
+  metadata.gz: 1021437d5c97ecfad14822548522cad49d3257992c91ee8ccf90a30e3cd4cd98fae6c44d977fc3c1cd4f61334bd7c935fcc50bdee92a68a4399470d4610836be
+  data.tar.gz: 5c805b9bb8b7cc1123be943966b2f4e94619580d17eb051947d38b1e3d4a165b2c61dd3f117c6bc7c8d858768731108bb1aab1cc59472ad0a3e120fb8df9cae4

data/lib/wayback_archiver.rb

@@ -4,19 +4,36 @@ require 'uri'
 require 'net/http'
 
 require 'wayback_archiver/version'
-require 'wayback_archiver/collector'
+require 'wayback_archiver/url_collector'
 require 'wayback_archiver/archive'
 require 'wayback_archiver/request'
 
+# WaybackArchiver, send URLs to Wayback Machine. By crawling, sitemap, file or single URL.
 module WaybackArchiver
-  def self.archive(source, from = :crawl)
-    urls = case from.to_s
-    when 'file'    then Archive.post(Collector.urls_from_file(source))
-    when 'crawl'   then Collector.urls_from_crawl(source) { |url| Archive.post_url(url) }
-    when 'sitemap' then Archive.post(Collector.urls_from_sitemap("#{source}/sitemap.xml"))
+  # Link to gem on rubygems.org, part of the sent User-Agent
+  INFO_LINK  = 'https://rubygems.org/gems/wayback_archiver'
+  # WaybackArchiver User-Agent
+  USER_AGENT = "WaybackArchiver/#{WaybackArchiver::VERSION} (+#{INFO_LINK})"
+
+  # Send URLs to Wayback Machine.
+  # @return [Array] with URLs sent to the Wayback Machine.
+  # @param [String] source for URL(s).
+  # @param [String/Symbol] type of source. Supported types: ['crawl', 'sitemap', 'url', 'file'].
+  # @example Crawl example.com and send all URLs of the same domain
+  #    WaybackArchiver.archive('example.com') # Default type is :crawl
+  #    WaybackArchiver.archive('example.com', :crawl)
+  # @example Send only example.com
+  #    WaybackArchiver.archive('example.com', :url)
+  # @example Send URL on each line in specified file
+  #    WaybackArchiver.archive('/path/to/file', :file)
+  def self.archive(source, type = :crawl)
+    case type.to_s
+    when 'file'    then Archive.post(UrlCollector.file(source))
+    when 'crawl'   then UrlCollector.crawl(source) { |url| Archive.post_url(url) }
+    when 'sitemap' then Archive.post(UrlCollector.sitemap(source))
     when 'url'     then Archive.post_url(Request.resolve_url(source))
     else
-      raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemap, url, file, crawl"
+      raise ArgumentError, "Unknown type: '#{type}'. Allowed types: sitemap, url, file, crawl"
     end
   end
 end

data/lib/wayback_archiver/archive.rb

@@ -1,27 +1,47 @@
 module WaybackArchiver
+  # Post URL(s) to Wayback Machine
   class Archive
-    WAYBACK_BASE_URL = 'https://web.archive.org/save/'
-    MAX_THREAD_COUNT = 10
+    # Wayback Machine base URL.
+    WAYBACK_BASE_URL    = 'https://web.archive.org/save/'
+    # Default concurrency for archiving URLs
+    DEFAULT_CONCURRENCY = 10
+    class << self
+      # Send URLs to Wayback Machine.
+      # @return [Array] with sent URLs.
+      # @param [Array] urls URLs to send.
+      # @param [Hash] options
+      # @example Archive example.com, with default options
+      #    Archive.post(['http://example.com'])
+      # @example Archive example.com, using only 1 thread
+      #    Archive.post(['http://example.com'], concurrency: 1)
+      def post(urls, options = {})
+        options     = { concurrency: DEFAULT_CONCURRENCY }.merge!(options)
+        concurrency = options[:concurrency]
+        puts "Request are sent with up to #{concurrency} parallel threads"
+        puts "Total urls to be sent: #{urls.length}"
+        group_size = (urls.length / concurrency) + 1
+        urls.each_slice(group_size).to_a.map! do |archive_urls|
+          Thread.new { archive_urls.each { |url| post_url(url) } }
+        end.each(&:join)
+        puts "#{urls.length} URLs sent to Internet archive"
+        urls
+      end
 
-    def self.post(urls)
-      puts "Request are sent with up to #{MAX_THREAD_COUNT} parallel threads"
-      puts "Total urls to be sent: #{urls.length}"
-      group_size = (urls.length / MAX_THREAD_COUNT) + 1
-      urls.each_slice(group_size).to_a.map! do |archive_urls|
-        Thread.new { archive_urls.each { |url| post_url(url) } }
-      end.each(&:join)
-      puts "#{urls.length} URLs sent to Internet archive"
-      urls
-    end
-
-    def self.post_url(archive_url)
-      resolved_url = Request.resolve_url(archive_url)
-      request_url  = "#{WAYBACK_BASE_URL}#{resolved_url}"
-      response     = Request.get_response(request_url)
-      puts "[#{response.code}, #{response.message}] #{resolved_url}"
-    rescue Exception => e
-      puts "Error message:     #{e.message}"
-      puts "Failed to archive: #{resolved_url}"
+      # Send URL to Wayback Machine.
+      # @return [String] the sent URL.
+      # @param [String] url to send.
+      # @example Archive example.com, with default options
+      #    Archive.post_url('http://example.com')
+      def post_url(url)
+        resolved_url = Request.resolve_url(url)
+        request_url  = "#{WAYBACK_BASE_URL}#{resolved_url}"
+        response     = Request.response(request_url)
+        puts "[#{response.code}, #{response.message}] #{resolved_url}"
+        resolved_url
+      rescue Exception => e
+        puts "Error message:     #{e.message}"
+        puts "Failed to archive: #{resolved_url}"
+      end
     end
   end
 end

data/lib/wayback_archiver/request.rb

@@ -1,34 +1,65 @@
 require 'url_resolver' # TODO: Allow users to use any resolver
 
 module WaybackArchiver
+  # Request and parse HTML & XML documents
   class Request
-    INFO_LINK  = 'https://rubygems.org/gems/wayback_archiver'
-    USER_AGENT = "WaybackArchiver/#{WaybackArchiver::VERSION} (+#{INFO_LINK})"
-
     class << self
-      def get_page(url, document_type = :html)
-        response = Request.get_response(url).body
-        case document_type
-        when :xml
-          Nokogiri::XML(response)
-        else
-          Nokogiri::HTML(response)
-        end
+      # Get and parse HTML & XML documents.
+      # @return [Array] with links sent to the Wayback Machine.
+      # @param [String] url to retrieve and parse.
+      # @example Request and parse example.com
+      #    Request.document('example.com')
+      # @example Request and parse google.com/sitemap.xml
+      #    Request.document('google.com/sitemap.xml')
+      def document(url)
+        response_body = Request.response(url).body
+        Nokogiri::HTML(response_body)
       end
 
-      def get_response(url, resolve = false)
+      # Get reponse.
+      # @return [Net::HTTP*] the http response.
+      # @param [String] url URL to retrieve.
+      # @param [Boolean] resolve whether to resolve the URL.
+      # @example Resolve example.com and request
+      #    Request.response('example.com', true)
+      # @example Request http://example.com
+      #    Request.response('http://example.com', false)
+      def response(url, resolve = true)
         resolved_url = resolve ? resolve_url(url) : url
         uri          = URI.parse(resolved_url)
         http         = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = true if resolved_url.include?('https://')
+        http.use_ssl = true if resolved_url.start_with?('https://')
 
         request = Net::HTTP::Get.new(uri.request_uri)
-        request['User-Agent'] = USER_AGENT
+        request['User-Agent'] = WaybackArchiver::USER_AGENT
         http.request(request)
       end
 
+      # Resolve the URL, follows redirects.
+      # @return [String] the resolved URL.
+      # @param [String] url to retrieve.
+      # @example Resolve example.com and request
+      #    Request.resolve_url('example.com')
       def resolve_url(url)
-        UrlResolver.resolve(url)
+        resolved = UrlResolver.resolve(url)
+        resolved = resolved.prepend('http://') unless has_protocol?(resolved)
+        resolved
+      end
+
+      private
+
+      # Resolve the URL, follows redirects.
+      # @return [Boolean] true if string includes protocol.
+      # @param [String] url to check.
+      # @example Check if string includes protocol
+      #    Request.has_protocol?('example.com')
+      #    # => false
+      #    Request.has_protocol?('https://example.com')
+      #    # => true
+      #    Request.has_protocol?('http://example.com')
+      #    # => true
+      def has_protocol?(url)
+        url.start_with?('http://') || url.start_with?('https://')
       end
     end
   end

data/lib/wayback_archiver/url_collector.rb

@@ -0,0 +1,40 @@
+module WaybackArchiver
+  # Retrive URLs from different sources
+  class UrlCollector
+    class << self
+      # Retrieve URLs from Sitemap.
+      # @return [Array] of URLs defined in Sitemap.
+      # @param [String] url domain to retrieve Sitemap from.
+      # @example Get URLs defined in Sitemap for google.com
+      #    UrlCollector.sitemap('https://google.com')
+      def sitemap(url)
+        resolved = Request.resolve_url("#{url}/sitemap.xml")
+        sitemap  = Request.document(resolved)
+        sitemap.css('loc').map { |element| element.text }
+      end
+
+      # Retrieve URLs by crawling.
+      # @return [Array] of URLs defined found during crawl.
+      # @param [String] url domain to crawl URLs from.
+      # @example Crawl URLs defined on example.com
+      #    UrlCollector.crawl('http://example.com')
+      def crawl(url)
+        SiteMapper.map(url, user_agent: WaybackArchiver::USER_AGENT) { |new_url| yield(new_url) if block_given? }
+      end
+
+      # Retrieve URLs listed in file.
+      # @return [Array] of URLs defined in file.
+      # @param [String] path to get URLs from.
+      # @example Get URLs defined in /path/to/file
+      #    UrlCollector.file('/path/to/file')
+      def file(path)
+        raise ArgumentError, "No such file: #{path}" unless File.exist?(path)
+        urls = []
+        File.open(path).read
+          .gsub(/\r\n?/, "\n")
+          .each_line { |line| urls << line.gsub(/\n/, '').strip }
+        urls.reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/wayback_archiver/version.rb

@@ -1,3 +1,4 @@
 module WaybackArchiver
-  VERSION = '0.0.10'
+  # Gem version
+  VERSION = '0.0.11'
 end

metadata

@@ -1,113 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-22 00:00:00.000000000 Z
+date: 2015-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: site_mapper
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: url_resolver
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 description: Send URLs to Wayback Machine. By crawling, sitemap, file or single URL.
 email:
 - burenstam@gmail.com
@@ -117,11 +131,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/wayback_archiver
+- lib/wayback_archiver.rb
 - lib/wayback_archiver/archive.rb
-- lib/wayback_archiver/collector.rb
 - lib/wayback_archiver/request.rb
+- lib/wayback_archiver/url_collector.rb
 - lib/wayback_archiver/version.rb
-- lib/wayback_archiver.rb
 homepage: https://github.com/buren/wayback_archiver
 licenses:
 - MIT
@@ -132,17 +146,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Send URLs to Wayback Machine

data/lib/wayback_archiver/collector.rb

@@ -1,24 +0,0 @@
-module WaybackArchiver
-  class Collector
-    class << self
-      def urls_from_sitemap(url)
-        resolved = Request.resolve_url(url)
-        sitemap  = Request.get_page(resolved)
-        sitemap.css('loc').map! { |element| element.text }
-      end
-
-      def urls_from_crawl(url)
-        SiteMapper.map(url) { |new_url| yield(new_url) if block_given? }
-      end
-
-      def urls_from_file(path)
-        raise ArgumentError, "No such file: #{path}" unless File.exist?(path)
-        urls = []
-        text = File.open(path).read
-        text.gsub!(/\r\n?/, "\n")
-            .each_line { |line| urls << line.gsub!(/\n/, '').strip }
-        urls.reject(&:empty?)
-      end
-    end
-  end
-end