wayback_archiver 0.0.3 → 0.0.4

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NjcyNWE4NTU0MDY1NDYwMTZiY2RkNTkyMTM2ZjE3NTg5YTVhZjdjMA==
-  data.tar.gz: !binary |-
-    YmQzYmEwMGVjMGQwNWY4NmNmOWZhY2MwNDIyNmM0MjBjNzg3YmM4ZA==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    NzdhNDM5NTI2Njk5YzNmZjNmZTAzMThmZThiOTJlZGZkMWEyMmExMzE5NDli
-    ZDgzOWE1ZGVjY2FkOTI1M2MxOGQ3Mjc0YzY5NWQyNTRhZWJmOTJhOTBhYjM3
-    YTc0YTAwZDUzYjcwMjdiNzdjYTQzY2I2NmQ4Y2U3Mzk0NTY3NDk=
-  data.tar.gz: !binary |-
-    YTBlY2ViNGE4OWFjMWIwNmQzNTM4ODMzMGI1MjkyNjMzNzMwMmQ4NDJhYjBk
-    YTk1MjYxYzQzOTZiZDQzODZhZGM4NGRkYjZmMDBiZWJkZTkwN2RlZTZjOWUz
-    NDVkMjc1N2E2ODdiZGI0OTMwZjAzMmYwZjQyNDg5MGZhNmNlN2Y=
+SHA1:
+  metadata.gz: 66f995653260c6d884271708691177c6c4e1c5b2
+  data.tar.gz: b46518ada2f179c8034bb48c41f17195e5f4551d
+SHA512:
+  metadata.gz: 5837775a0b99935895992589a75490859e4b3c2b089ee20e3d339e52913cfc56c1e5e055bd4650bb5678b93abf76f2d7204a27d67ea1e9403c69f20cdab1d422
+  data.tar.gz: 5d256b29a98bfba85dad1559075885fa0b85e9c632971a735207fff7c636aede116b45581cdf72f8a3054cb351f84ae1556ce1849a0f33738989a108bba7f8f1

data/lib/wayback_archiver/archive.rb

@@ -1,18 +1,29 @@
 module WaybackArchiver
   class Archive
+    MAX_THREAD_COUNT = 8
 
-    def self.post(urls)
-      urls.each_with_index do |url, index|
-        request_url = "#{BASE_URL}#{url}"
-        puts "Archiving (#{index + 1}/#{urls.length}): #{url}"
-        begin
-          res = Request.get_response(request_url)
-          puts "#{res.code} => #{res.message}"
-        rescue Exception => e
-          puts "Error message: #{e.message}"
-          puts "Failed to archive: #{url}"
+    def self.post(all_urls)
+      puts "Request will be sent with max #{MAX_THREAD_COUNT} parallel threads"
+  
+      puts "Total urls to be sent: #{all_urls.length}"
+      threads    = Array.new
+      group_size = (all_urls.length / MAX_THREAD_COUNT) + 1
+
+      all_urls.each_slice(group_size).to_a.each do |urls|
+        threads << Thread.new do
+          urls.each_with_index do |url, index|
+            request_url = "#{BASE_URL}#{url}"
+            begin
+              res = Request.get_response(request_url)
+              print "#{url}    #{res.code} => #{res.message} \n"
+            rescue Exception => e
+              puts "Error message: #{e.message}"
+              puts "Failed to archive: #{url}"
+            end
+          end
         end
       end
+      threads.each(&:join)
     end
 
   end

data/lib/wayback_archiver/request.rb

@@ -7,7 +7,7 @@ module WaybackArchiver
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = true if url.include?('https://')
 
-      request   = Net::HTTP::Get.new(uri.request_uri)
+      request  = Net::HTTP::Get.new(uri.request_uri)
       response = http.request(request)
       response
     end

data/lib/wayback_archiver/version.rb

@@ -1,3 +1,3 @@
 module WaybackArchiver
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Jacob Burenstam
@@ -28,17 +28,17 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: ! 'Send URLs to Wayback Machine. From: sitemap, file or single URL.'
+description: 'Send URLs to Wayback Machine. From: sitemap, file or single URL.'
 email:
 - burenstam@gmail.com
 executables:
@@ -48,9 +48,9 @@ extra_rdoc_files: []
 files:
 - bin/wayback_archiver
 - lib/wayback_archiver/archive.rb
-- lib/wayback_archiver/version.rb
 - lib/wayback_archiver/collector.rb
 - lib/wayback_archiver/request.rb
+- lib/wayback_archiver/version.rb
 - lib/wayback_archiver.rb
 homepage: https://github.com/buren/wayback_archiver
 licenses:
@@ -62,17 +62,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
 summary: Send URLs to Wayback Machine