filbunke 1.4.1 → 1.5.0

data/Rakefile

@@ -14,6 +14,7 @@ begin
     gem.files.exclude 'pkg'
     gem.executables = ['filbunked']
     gem.add_dependency 'json', '>= 1.1.7'
+    gem.add_dependency 'typhoeus', '>= 0.2.0'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-1.4.1
+1.5.0

data/filbunke.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{filbunke}
-  s.version = "1.4.1"
+  s.version = "1.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Wouter de Bie"]
-  s.date = %q{2010-12-02}
+  s.date = %q{2010-12-03}
   s.default_executable = %q{filbunked}
   s.description = %q{Filbunke client and library}
   s.email = %q{wouter@deltaprojects.se}
@@ -48,13 +48,16 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_runtime_dependency(%q<json>, [">= 1.1.7"])
+      s.add_runtime_dependency(%q<typhoeus>, [">= 0.2.0"])
     else
       s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_dependency(%q<json>, [">= 1.1.7"])
+      s.add_dependency(%q<typhoeus>, [">= 0.2.0"])
     end
   else
     s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
     s.add_dependency(%q<json>, [">= 1.1.7"])
+    s.add_dependency(%q<typhoeus>, [">= 0.2.0"])
   end
 end
 

data/lib/filbunke/client.rb

@@ -26,7 +26,10 @@ module Filbunke
       new_checkpoint = updates["checkpoint"]
       
       @logger.log "Updating repository: #{repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}" if updated_files.size > 0
-      
+
+      @hydra = Typhoeus::Hydra.new
+      @async_requests = []
+
       updated_files.each do |raw_file|
         file = File.new(raw_file)
 
@@ -39,9 +42,7 @@ module Filbunke
           if file_needs_update?(file, local_file_path)
           
             if update_file!(file, local_file_path) then
-              @callbacks.each do |callback|
-                callback.on_update(file)
-              end
+
               yield file
             else
               @logger.log "Unable to get file #{file.url} ==> #{file.path}!"
@@ -53,6 +54,10 @@ module Filbunke
           end
         end
       end
+
+      @hydra.run
+      failure = failure || @async_requests.any?{|request| request.handled_response == false }
+
       failure ? last_checkpoint : (new_checkpoint || last_checkpoint)
     end
     
@@ -120,11 +125,19 @@ module Filbunke
         update_http_file!(file, local_file_path)
       elsif file.url =~ /^hdfs:\/\//
         update_hdfs_file!(file, local_file_path)
+        run_callbacks(file)
+
       else
         raise "Unsupported protocol for file: #{file.inspect}"
       end
     end
 
+    def run_callbacks(file)
+      @callbacks.each do |callback|
+        callback.on_update(file)
+      end
+    end
+
     def file_needs_update?(file, local_file_path)
       return true if file.hash.nil? || file.hash == ""
 
@@ -161,26 +174,37 @@ module Filbunke
     end
 
     def update_http_file!(file, local_file_path)
-      begin 
-        uri = URI.parse(file.url)
-        file_http=Net::HTTP.new(uri.host, uri.port)
-        file_http.start do |http|
-          request = Net::HTTP::Get.new(uri.path)
-          request.basic_auth @repository.user, @repository.pass if @repository.user
-          response = http.request(request)
-          if response.code.to_i == 200
-            write_file!(local_file_path, response.body)
-          elsif response.code.to_i == 404
-            delete_file!(local_file_path)
-          else
-            @logger.log "Failed to update file #{uri}, error code = #{response.code}"
+      begin
+        async_request = if @repository.user
+          Typhoeus::Request.new(file.url)
+        else
+          Typhoeus::Request.new(file.url, :user => @repository.user, :password => @repository.pass)
+        end
+        async_request.on_complete do |response|
+          success = true
+          begin
+            if response.code.to_i == 200
+              write_file!(local_file_path, response.body)
+            elsif response.code.to_i == 404
+              delete_file!(local_file_path)
+            else
+              @logger.log "Failed to update file #{file.url}, error code = #{response.code}"
+              success = false
+            end
+            run_callbacks(file) if success
+          rescue StandardError => e
+            @logger.log "Failed to update file #{file.url}: #{e.message}"
+            success = false
           end
-          return true
+          success
         end
+        @hydra.queue async_request
+        @async_requests << async_request
       rescue StandardError => e
-        @logger.log "Failed to update file #{uri}: #{e.message}"
+        @logger.log "Failed to update file #{file.url}: #{e.message}"
         return false
       end
+      return true
     end
     
     def update_hdfs_file!(file, local_file_path)

data/lib/filbunke.rb

@@ -3,6 +3,7 @@ require 'json'
 require 'net/http'
 require 'fileutils'
 require 'digest/md5'
+require 'typhoeus'
 
 require 'filbunke/client.rb'
 require 'filbunke/file.rb'

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 4
-  - 1
-  version: 1.4.1
+  - 5
+  - 0
+  version: 1.5.0
 platform: ruby
 authors: 
 - Wouter de Bie
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-02 00:00:00 +01:00
+date: 2010-12-03 00:00:00 +01:00
 default_executable: filbunked
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -43,6 +43,20 @@ dependencies:
         version: 1.1.7
   type: :runtime
   version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: typhoeus
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  type: :runtime
+  version_requirements: *id003
 description: Filbunke client and library
 email: wouter@deltaprojects.se
 executables: