wayback_archiver 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ab0900a297f8e0413c436d9cf084ebfd66c09a9
-  data.tar.gz: 8801173b1f3bd9d10e6184bcc185056a835c30d9
+  metadata.gz: 3df67fcb7faa06087ad72e6c57ff985d9dbe4fe9
+  data.tar.gz: 5c21d0bf3353dd2ce732cef3b59cd3ba807acc8a
 SHA512:
-  metadata.gz: fb35d8b324df42b37aaa10b3fbf90428f9d227800db935d64a9bf674a64f438c64e24a1ba0a529a6171edae144300ea6d991e8591b278e3373d3e5c3b8b1869a
-  data.tar.gz: b9c18c4062d056e4e6a766b337701deae9178124da8455631ad45a10350a6498f885ad44d6c6757246ecf43653b6b48d94254e9c768b8df37a91f141887f41fc
+  metadata.gz: d21d169fb9fa143b777d771a8cdc5ba8c3c4186f6472aa8f2ff2806034becf18e3cc260c7ce0a9b97d1148a2d1781d9ddc0daef08d7abe1d2b13140125513231
+  data.tar.gz: 15a413ab5bd0f51c40640cd0be764142ca0380315f97fc335cf1bf19f4645b327dc0716400c6c10af7a79b8ceb898b6e0f89a21114865af7f84da1e2bf3d383c

data/lib/wayback_archiver.rb

@@ -1,22 +1,22 @@
+require 'site_mapper'
+
 require 'uri'
 require 'net/http'
 
+require 'wayback_archiver/version'
 require 'wayback_archiver/collector'
 require 'wayback_archiver/archive'
 require 'wayback_archiver/request'
-require 'wayback_archiver/crawler'
-require 'wayback_archiver/crawl_url'
 
 module WaybackArchiver
   def self.archive(source, from = :crawl)
     urls = case from.to_s
-    when 'sitemap' then Collector.urls_from_sitemap("#{source}/sitemap.xml")
-    when 'url'     then [Request.resolve_url(source)]
-    when 'file'    then Collector.urls_from_file(source)
-    when 'crawl'   then Collector.urls_from_crawl(source)
+    when 'file'    then Archive.post(Collector.urls_from_file(source))
+    when 'crawl'   then Collector.urls_from_crawl(source) { |url| Archive.post_url(url) }
+    when 'sitemap' then Archive.post(Collector.urls_from_sitemap("#{source}/sitemap.xml"))
+    when 'url'     then Archive.post_url(Request.resolve_url(source))
     else
       raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemap, url, file, crawl"
     end
-    Archive.post(urls)
   end
 end

data/lib/wayback_archiver/archive.rb

@@ -7,7 +7,7 @@ module WaybackArchiver
       puts "Request are sent with up to #{MAX_THREAD_COUNT} parallel threads"
       puts "Total urls to be sent: #{urls.length}"
       group_size = (urls.length / MAX_THREAD_COUNT) + 1
-      urls.each_slice(group_size).to_a.map do |archive_urls|
+      urls.each_slice(group_size).to_a.map! do |archive_urls|
         Thread.new { archive_urls.each { |url| post_url(url) } }
       end.each(&:join)
       puts "#{urls.length} URLs sent to Internet archive"

data/lib/wayback_archiver/collector.rb

@@ -4,19 +4,19 @@ module WaybackArchiver
       def urls_from_sitemap(url)
         resolved = Request.resolve_url(url)
         sitemap  = Request.get_page(resolved)
-        sitemap.css('loc').map { |element| element.text }
+        sitemap.css('loc').map! { |element| element.text }
       end
 
       def urls_from_crawl(url)
-        Crawler.collect_urls(url)
+        SiteMapper.map(url) { |new_url| yield(new_url) if block_given? }
       end
 
       def urls_from_file(path)
         raise ArgumentError, "No such file: #{path}" unless File.exist?(path)
         urls = []
         text = File.open(path).read
-        text.gsub!(/\r\n?/, "\n") # Normalize line endings
-        text.each_line { |line| urls << line.gsub(/\n/, '').strip }
+        text.gsub!(/\r\n?/, "\n")
+            .each_line { |line| urls << line.gsub!(/\n/, '').strip }
         urls.reject(&:empty?)
       end
     end

data/lib/wayback_archiver/request.rb

@@ -3,7 +3,7 @@ require 'url_resolver' # TODO: Allow users to use any resolver
 module WaybackArchiver
   class Request
     INFO_LINK  = 'https://rubygems.org/gems/wayback_archiver'
-    USER_AGENT = "WaybackArchiver/#{VERSION} (+#{INFO_LINK})"
+    USER_AGENT = "WaybackArchiver/#{WaybackArchiver::VERSION} (+#{INFO_LINK})"
 
     class << self
       def get_page(url, document_type = :html)

data/lib/wayback_archiver/version.rb

@@ -1,3 +1,3 @@
 module WaybackArchiver
-  VERSION = '0.0.9'
+  VERSION = '0.0.10'
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-21 00:00:00.000000000 Z
+date: 2014-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: site_mapper
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: url_resolver
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -56,59 +56,59 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-description: 'Send URLs to Wayback Machine. From: sitemap, file or single URL.'
+        version: '0.7'
+description: Send URLs to Wayback Machine. By crawling, sitemap, file or single URL.
 email:
 - burenstam@gmail.com
 executables:
@@ -119,8 +119,6 @@ files:
 - bin/wayback_archiver
 - lib/wayback_archiver/archive.rb
 - lib/wayback_archiver/collector.rb
-- lib/wayback_archiver/crawl_url.rb
-- lib/wayback_archiver/crawler.rb
 - lib/wayback_archiver/request.rb
 - lib/wayback_archiver/version.rb
 - lib/wayback_archiver.rb
@@ -136,7 +134,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='

data/lib/wayback_archiver/crawl_url.rb

@@ -1,74 +0,0 @@
-module WaybackArchiver
-  class CrawlUrl
-    attr_reader :resolved_base_url, :base_hostname
-
-    def initialize(base_url)
-      @resolved_base_url = Request.resolve_url(base_url)
-      @base_hostname     = URI.parse(@resolved_base_url).hostname
-      @resolved_base_url.prepend('http://') unless @resolved_base_url.start_with?('http')
-    end
-
-    def absolute_url_from(raw_url, get_url)
-      return nil unless eligible_url?(raw_url)
-      parsed_url = URI.parse(raw_url) rescue URI.parse('')  
-      if parsed_url.relative?
-        url_from_relative(raw_url, get_url)
-      elsif same_domain?(raw_url, @resolved_base_url)
-        raw_url
-      else
-        nil
-      end
-    end
-
-    private
-
-    def url_from_relative(url, current_page_url)
-      if url.start_with?('/')
-        "#{without_path_suffix(resolved_base_url)}#{url}"
-      elsif url.start_with?('../')
-        "#{url_from_dotted_url(url, current_page_url)}"
-      else
-        "#{with_path_suffix(resolved_base_url)}#{url}"
-      end
-    end
-
-    def url_from_dotted_url(url, current_page_url)
-      absolute_url = with_path_suffix(current_page_url.dup)
-      found_dots   = without_path_suffix(url).scan('../').length
-      removed_dots = 0
-      max_levels   = 4
-      while found_dots >= removed_dots && max_levels > removed_dots
-        index = absolute_url.rindex('/') or break
-        absolute_url = absolute_url[0..(index - 1)]
-        removed_dots += 1
-      end
-      "#{with_path_suffix(absolute_url)}#{url.gsub('../', '')}"
-    end
-
-    def with_path_suffix(passed_url)
-      url = passed_url.dup
-      url.end_with?('/') ? url : url << '/'
-    end
-
-    def without_path_suffix(passed_url)
-      url = passed_url.dup
-      url.end_with?('/') ? url[0...(url.length - 1)] : url
-    end
-
-    def eligible_url?(href)
-      return false if href.nil? || href.empty?
-      dont_start   = %w(javascript: callto: mailto: tel: skype: facetime: wtai: #)
-      dont_include = %w(/email-protection#)
-      dont_end     = %w(.zip .rar .pdf .exe .dmg .pkg .dpkg .bat)
-
-      dont_start.each   { |pattern| return false if href.start_with?(pattern) }      
-      dont_include.each { |pattern| return false if href.include?(pattern) }
-      dont_end.each     { |pattern| return false if href.end_with?(pattern) }
-      true
-    end
-
-    def same_domain?(first, second)
-      first.include?(second)
-    end
-  end
-end

data/lib/wayback_archiver/crawler.rb

@@ -1,56 +0,0 @@
-require 'set'
-require 'nokogiri'
-
-module WaybackArchiver
-  class Crawler
-    CRAWLER_INFO_LINK = 'https://rubygems.org/gems/wayback_archiver'
-    HEADERS_HASH      = {
-      'User-Agent' => "WaybackArchiver/#{WaybackArchiver::VERSION} (+#{CRAWLER_INFO_LINK})"
-    }
-
-    def initialize(url, resolve = false)
-      base_url     = Request.resolve_url(url)
-      @options     = { resolve: resolve }
-      @crawl_url   = CrawlUrl.new(base_url)
-      @fetch_queue = Set.new
-      @processed   = Set.new
-      @fetch_queue << @crawl_url.resolved_base_url
-    end
-
-    def self.collect_urls(base_url)
-      new(base_url).collect_urls
-    end
-
-    def collect_urls
-      until @fetch_queue.empty?
-        url = @fetch_queue.first
-        @fetch_queue.delete(@fetch_queue.first)
-        page_links(url)
-      end
-      puts "Crawling finished, #{@processed.length} links found"
-      @processed.to_a
-    rescue Interrupt, IRB::Abort
-      puts 'Crawl interrupted.'
-      @fetch_queue.to_a
-    end
-
-    private
-
-    def page_links(get_url)
-      puts "Queue length: #{@fetch_queue.length}, Parsing: #{get_url}"
-      link_elements = Request.get_page(get_url).css('a') rescue []
-      @processed << get_url
-      link_elements.each do |page_link|
-        absolute_url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
-        if absolute_url
-          resolved_url = resolve(absolute_url)
-          @fetch_queue << resolved_url if !@processed.include?(resolved_url)
-        end
-      end
-    end
-
-    def resolve(url)
-      @options[:resolve] ? Request.resolve_url(url) : url
-    end
-  end
-end