wayback_archiver 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66f995653260c6d884271708691177c6c4e1c5b2
-  data.tar.gz: b46518ada2f179c8034bb48c41f17195e5f4551d
+  metadata.gz: 87e286b40825e15468832af25c42c98b59170b21
+  data.tar.gz: 9a7eb7e6fc6db882cf23878718626db04847b33a
 SHA512:
-  metadata.gz: 5837775a0b99935895992589a75490859e4b3c2b089ee20e3d339e52913cfc56c1e5e055bd4650bb5678b93abf76f2d7204a27d67ea1e9403c69f20cdab1d422
-  data.tar.gz: 5d256b29a98bfba85dad1559075885fa0b85e9c632971a735207fff7c636aede116b45581cdf72f8a3054cb351f84ae1556ce1849a0f33738989a108bba7f8f1
+  metadata.gz: 562641a2749e3a3e644ad7056f365b8d26566809b66606d1c6b95bf5c2db158039dd4a5b1c33677817c5f61d06786a63ec227150534a4dd5d1f958ccd1171dc9
+  data.tar.gz: d635ae3f69ab3e585bf0519389d9335edfe4848b4c6a41f271f957413bb663d930974f2fe386445fef6f7a5bb7ffc9d0d5c3f3bdf8849c0cc66081b7806d61a8

data/lib/wayback_archiver.rb

@@ -5,20 +5,23 @@ require 'rexml/document'
 require 'wayback_archiver/collector'
 require 'wayback_archiver/archive'
 require 'wayback_archiver/request'
+require 'wayback_archiver/crawler'
 
 module WaybackArchiver
   BASE_URL = 'https://web.archive.org/save/'
 
   def self.archive(source, from = :sitemap)
-    urls = case from
-    when :sitemap, 'sitemap'
+    urls = case from.to_s
+    when 'sitemap'
       Collector.urls_from_sitemap("#{source}/sitemap.xml")
-    when :url, 'url'
+    when 'url'
       Array(source)
-    when :file, 'file'
+    when 'file'
       Collector.urls_from_file(source)
+    when 'crawl', 'crawler'
+      Crawler.collect_urls(source)
     else
-      raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemal, url, file"
+      raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemap, url, file, crawler"
     end
     Archive.post(urls)
   end

data/lib/wayback_archiver/archive.rb

@@ -6,9 +6,8 @@ module WaybackArchiver
       puts "Request will be sent with max #{MAX_THREAD_COUNT} parallel threads"
   
       puts "Total urls to be sent: #{all_urls.length}"
-      threads    = Array.new
+      threads    = []
       group_size = (all_urls.length / MAX_THREAD_COUNT) + 1
-
       all_urls.each_slice(group_size).to_a.each do |urls|
         threads << Thread.new do
           urls.each_with_index do |url, index|
@@ -23,8 +22,17 @@ module WaybackArchiver
           end
         end
       end
-      threads.each(&:join)
+      threads.each_with_index do |thread, index|
+        print_index = index + 1
+        progress = '['
+        progress << '#' * print_index
+        progress << ' ' * (threads.length - print_index)
+        progress << ']'
+        procent = ((print_index.to_f/threads.length.to_f) * 100).round(0)
+        puts "[PROGRESS] #{progress} #{procent}% (#{print_index}/#{threads.length})"
+        thread.join
+      end
+      all_urls
     end
-
   end
 end

data/lib/wayback_archiver/crawler.rb

@@ -0,0 +1,50 @@
+require 'set'
+require 'nokogiri'   
+require 'open-uri'
+
+module WaybackArchiver
+  class Crawler
+    def initialize(base_url)
+      @base_url    = base_url
+      @hostname    = URI.parse(@base_url).host
+      @fetch_queue = Set.new
+      @procesed    = Set.new
+      @fetch_queue << @base_url
+    end
+
+    def self.collect_urls(base_url)
+      new(base_url).collect_urls
+    end
+
+    def collect_urls
+      until @fetch_queue.empty?
+        url = @fetch_queue.first
+        @fetch_queue.delete(@fetch_queue.first)
+        page_links(url)
+      end
+      puts "Crawling finished, #{@procesed.length} links found"
+      @procesed.to_a
+    end
+
+    def page_links(url)
+      puts "Queue length: #{@fetch_queue.length}, Parsing: #{url}"
+      link_elements = Nokogiri::HTML(open(url)).css('a') rescue []
+      @procesed << url
+      link_elements.each do |link|
+        href = sanitize_url(link.attr('href'))
+        @fetch_queue << href if href && !@procesed.include?(href)
+      end
+    end
+
+    def sanitize_url(raw_url)
+      url = URI.parse(raw_url) rescue URI.parse('')  
+      if url.host.nil?
+        sanitized_url  = "#{@base_url}#{url.path}"
+        sanitized_url += "?#{url.query}" unless url.query.nil?
+        sanitized_url
+      else
+        raw_url if raw_url.include?(@base_url) && @hostname.eql?(url.hostname)
+      end
+    end
+  end
+end

data/lib/wayback_archiver/version.rb

@@ -1,3 +1,3 @@
 module WaybackArchiver
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-11 00:00:00.000000000 Z
+date: 2014-10-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -49,6 +63,7 @@ files:
 - bin/wayback_archiver
 - lib/wayback_archiver/archive.rb
 - lib/wayback_archiver/collector.rb
+- lib/wayback_archiver/crawler.rb
 - lib/wayback_archiver/request.rb
 - lib/wayback_archiver/version.rb
 - lib/wayback_archiver.rb
@@ -77,3 +92,4 @@ signing_key:
 specification_version: 4
 summary: Send URLs to Wayback Machine
 test_files: []
+has_rdoc: