filbunke 1.12.0 → 1.13.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzE3ODU4ZGQzMmIyYzA1MzI0MmVkZTY4YWNkNTE2NDk3MGQ4Y2RmYw==
+    MTMwOWRhNTMzYzNjOTM5YTNlNzM4MDg1MmZiNzk1NDY4Yjk2ZmU0ZQ==
   data.tar.gz: !binary |-
-    YTQ1ZGNlYTVmYTNhY2RmY2UyNmFiZWVkYzIxZThmYTQ4NTlhZTVmMw==
+    Mzc2NWQ3YjAxOGJkZWQxYmI1ZDI2MmY2MmMzZjcwZjk5MDhlNWU4OA==
 SHA512:
   metadata.gz: !binary |-
-    Mzc2NmU2YzRhNDJhNzNkNDFhZTI2MDg2NzU3ZWU3MDkyZmM5ODU5ZTFlMDBi
-    Y2FhM2QxZGQ2ZjkyNmRmZGY0ODNlMzFmMmI1MzZlNGYzNTRjN2E0ODllNDhk
-    MWEzOTQ2OTFlZGQ1MDk3NTcwNDk5YzUzYzUyODliZjNiMmQxMGQ=
+    Y2Q3MTA0NjJkZGY5Zjg2ZTZmYjY0MjAxMmQ5NTEwNmIxMTUwNmVjNjMwMWNi
+    YTg4MDA0MjgzMjU1MDM5N2U3ZGYyMDg4MDc5NDA2NDlhNWE5OTk3NDlmNzgw
+    MWI2MDMxMjAwYjNhYTVlNzRjOTY4NWYxZjQ0YTI4MzAzZmMwNDc=
   data.tar.gz: !binary |-
-    NDRjNzVjN2FmZDgzODYzMjNiYzM4NGQ3YzkzNTJjNGVhMmU4M2NjN2Y3YTky
-    YzU2MmIzNDE4ZWNmZGJkM2NiZmQwNDk1MDQ5ZTkwNDE3M2JjNWY4ZTFhYmIz
-    N2IwMTE1MjQwM2M2M2RiOTM2YmJhOTQzNjcwNmUxYTkzNWMwODI=
+    ZDBjNWZkNGVhOGU1ZGI3ZDBjMmEzMzA3OTVkODExZTAyNzMyOWE1NmZlMTFh
+    YjZkZGZmNTlhZWNlYzY1NTg2NjY1MjhkNDlmNzY5ZTNkMDAxZTY3MzE0YmI3
+    YTdjNTE2OWNjMWE2ZjJmNmZjMTY2ZjIwOGVmYTRjNWI3NjgzNTk=

data/Rakefile

@@ -14,7 +14,7 @@ begin
     gem.files.exclude 'pkg'
     gem.executables = ['filbunked']
     gem.add_dependency 'json', '= 1.7.5'
-    gem.add_dependency 'typhoeus', '= 0.3.3'
+    gem.add_dependency 'typhoeus', '= 1.0.1'
     gem.add_dependency 'open4', '= 1.3.0'
     gem.add_dependency 'mime-types', '= 1.19'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings

data/VERSION

@@ -1 +1 @@
-1.12.0
+1.13.0

data/filbunke.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: filbunke 1.12.0 ruby lib
+# stub: filbunke 1.13.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "filbunke"
-  s.version = "1.12.0"
+  s.version = "1.13.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
-  s.date = "2016-03-04"
+  s.date = "2016-03-05"
   s.description = "Filbunke client and library"
   s.email = "technical@deltaprojects.com"
   s.executables = ["filbunked"]
@@ -52,20 +52,20 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_runtime_dependency(%q<json>, ["= 1.7.5"])
-      s.add_runtime_dependency(%q<typhoeus>, ["= 0.3.3"])
+      s.add_runtime_dependency(%q<typhoeus>, ["= 1.0.1"])
       s.add_runtime_dependency(%q<open4>, ["= 1.3.0"])
       s.add_runtime_dependency(%q<mime-types>, ["= 1.19"])
     else
       s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_dependency(%q<json>, ["= 1.7.5"])
-      s.add_dependency(%q<typhoeus>, ["= 0.3.3"])
+      s.add_dependency(%q<typhoeus>, ["= 1.0.1"])
       s.add_dependency(%q<open4>, ["= 1.3.0"])
       s.add_dependency(%q<mime-types>, ["= 1.19"])
     end
   else
     s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
     s.add_dependency(%q<json>, ["= 1.7.5"])
-    s.add_dependency(%q<typhoeus>, ["= 0.3.3"])
+    s.add_dependency(%q<typhoeus>, ["= 1.0.1"])
     s.add_dependency(%q<open4>, ["= 1.3.0"])
     s.add_dependency(%q<mime-types>, ["= 1.19"])
   end

data/lib/filbunke/client.rb

@@ -25,18 +25,17 @@ module Filbunke
     def with_updated_files(last_checkpoint)
       updates = get_updated_file_list(last_checkpoint)
       updated_files = updates["files"] || []
-      failure = false
-      
-      new_checkpoint = updates["checkpoint"]
-      
-      @logger.info "Updating repository: #{@repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}" if updated_files.size > 0
+      new_checkpoint = updates["checkpoint"] || 0
+      if updated_files.empty?
+        return new_checkpoint
+      end
 
+      @logger.info "Updating repository: #{@repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}"
       @async_requests = []
-      
       callbacks_on_update = []
       callbacks_on_no_change = []
       callbacks_on_delete = []
-
+      has_update_file_failure = false
       updated_files.each do |raw_file|
         file = File.new(raw_file)
         local_file_path = ::File.join(@repository.local_path, file.path)
@@ -50,7 +49,8 @@ module Filbunke
               callbacks_on_update << OpenStruct.new({ :file => file, :local_file_path => local_file_path })
             else
               @logger.error "Unable to get file #{file.url} ==> #{file.path}!"
-              failure = true
+              has_update_file_failure = true
+              break
             end
           else
             @logger.debug "File exists with correct hash: #{local_file_path}"
@@ -58,35 +58,47 @@ module Filbunke
           end
         end
       end
-      @hydra.run
 
-      pfailure = failure || @async_requests.any? do |request| 
-        @logger.warn "request did not handle response: #{request.inspect}" if request.response.nil? || request.response.code != 200
-        request.response.nil? || request.response.code != 200
+      if has_update_file_failure
+        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
+        return last_checkpoint
       end
-
-      if pfailure == false
-        @logger.info "Done fetching files for #{@repository.name}, processing callbacks..."
-        begin
-          
-          run_callbacks_delete(callbacks_on_delete)
-          
-          run_callbacks(callbacks_on_update)
-          
-          run_callbacks_no_change(callbacks_on_no_change)
-
-          new_checkpoint || last_checkpoint
-        rescue RuntimeError, SystemCallError, StandardError => e
-          msg = ["Callbacks failed to run; #{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-          @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
-          last_checkpoint
+      @logger.info "Done setting up async requests for #{@repository.name},  starting fetch..."
+      has_fetch_failures = begin
+        @hydra.run
+        # Magnus 20160305 - since we now fail fast by raising a RuntimeError on response.code != 200
+        # I think we can remove the following request validation
+        @async_requests.any? do |request|
+          @logger.warn "request did not handle response: #{request.inspect}" if request.response.nil? || request.response.code != 200
+          request.response.nil? || request.response.code != 200
         end
-      else
-        @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
+      rescue RuntimeError, SystemCallError, StandardError => e
+        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
+        true
+      end
+
+      if has_fetch_failures
+        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
+        return last_checkpoint
+      end
+
+      @logger.info "Done fetching files for #{@repository.name}, processing callbacks..."
+      new_or_last_checkpoint = begin
+        run_callbacks_delete(callbacks_on_delete)
+        run_callbacks(callbacks_on_update)
+        run_callbacks_no_change(callbacks_on_no_change)
+
+        new_checkpoint || last_checkpoint
+      rescue RuntimeError, SystemCallError, StandardError => e
+        msg = ["Callbacks failed to run; #{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "FAILED to process callbacks for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
         last_checkpoint
       end
+
+      new_or_last_checkpoint
     end
-    
+
     def update_files!(last_checkpoint)
       with_updated_files(last_checkpoint) {}
     end
@@ -150,7 +162,7 @@ module Filbunke
         return response.body.chomp.to_i
       end
     end
-    
+
     private
 
     def log_failed_request(failed_request_command, e)
@@ -162,11 +174,11 @@ module Filbunke
     end
 
     def update_file!(file, local_file_path)
-        
+
       if file.url =~ /^http:\/\//
         update_http_file!(file, local_file_path)
       elsif (file.url =~ /^hdfs:\/\//)
-        success = update_hdfs_file!(file, local_file_path)
+        update_hdfs_file!(file, local_file_path)
       else
         raise "Unsupported protocol for file: #{file.inspect}"
       end
@@ -205,7 +217,7 @@ module Filbunke
         updates_http.read_timeout = 300 # default is 60 seconds
         updates_http.start do |http|
           updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}"
-          updates_path = "#{updates_path}&batch_size=#{@repository.batch_size}" if @repository.batch_size
+          updates_path = "#{updates_path}&batch_size=#{@repository.batch_size}" if @repository.batch_size > 0
           begin
             @logger.info "Fetching updated file list from #{updates_path}"
             request = Net::HTTP::Get.new(updates_path)
@@ -224,44 +236,58 @@ module Filbunke
       rescue StandardError => e
         @logger.error "Unable to create HTTP connection to #{@repository.host}:#{@repository.port} (#{e.message})!"
         return {}
-      end        
+      end
     end
 
     def update_http_file!(file, local_file_path)
       begin
         async_request = if @repository.user
-          Typhoeus::Request.new(URI.encode(file.url), :follow_location => true, :username => @repository.user, :password => @repository.pass)
+          Typhoeus::Request.new(
+            URI.escape(file.url),
+            :followlocation => true,
+            :username => @repository.user,
+            :password => @repository.pass
+          )
         else
-          Typhoeus::Request.new(URI.encode(file.url), :follow_location => true)
+          Typhoeus::Request.new(
+            URI.escape(file.url),
+            :followlocation => true
+          )
+        end
+
+        downloaded_file = nil
+        async_request.on_headers do |response|
+          if response.code != 200
+            raise "Failed to fetch response(#{response.code}) for url '#{response.effective_url}' ---\n\t #{response.inspect}"
+          end
+          @logger.debug("Updating: #{local_file_path}")
+          ::FileUtils.mkdir_p(::File.dirname(local_file_path))
+          downloaded_file = ::File.new("#{local_file_path}.tmp", "wb")
         end
+
+        async_request.on_body do |chunk, response|
+          downloaded_file.write(chunk) if response.code == 200
+        end
+
         async_request.on_complete do |response|
-          success = false
-          begin
-            success = response.code.to_i == 200
-            if success
-              write_file!(local_file_path, response.body)
-            else
-              body_if_error = response.code >= 500 ? ", body = #{response.body}" : ""
-              @logger.warn "Failed to update file #{file.url}, got status code = #{response.code}#{body_if_error}"
-            end
-          rescue SystemCallError, StandardError => e
-            msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-            @logger.error "Failed to update file #{file.url}: #{msg}"
+          unless downloaded_file.nil?
+            downloaded_file.close
+            ::FileUtils.mv("#{local_file_path}.tmp", local_file_path)
+          else
+            ::FileUtils.rm("#{local_file_path}.tmp") if ::File.exist?("#{local_file_path}.tmp")
           end
-          # return the async_request.handled_response value here
-          success
+          true
         end
         @hydra.queue async_request
         @async_requests << async_request
-      rescue StandardError => e
+        true
+      rescue RuntimeError, SystemCallError, StandardError => e
         msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
         @logger.error "Failed to update file #{file.url}: #{msg}"
-        return false
+        false
       end
-
-      return true
     end
-    
+
     def update_hdfs_file!(file, local_file_path)
       begin
         ::FileUtils.mkdir_p(::File.dirname(local_file_path))
@@ -270,10 +296,10 @@ module Filbunke
         url.gsub!(/hdfs:\/\/([^\/]*)(.*)/, "hdfs://\\2")
         hdfs_cmd = "#{@repository.hadoop_binary} dfs -copyToLocal #{url} #{local_file_path}.tmp"
         #@logger.debug "Trying to update #{local_file_path} with '#{hdfs_cmd}'"
-        
+
         pid, stdin, stdout, stderr = Open4::popen4 hdfs_cmd
         ignored, status = Process::waitpid2 pid
-              
+
         if status.exitstatus == 0 then
           begin
             ::FileUtils.mv "#{local_file_path}.tmp", local_file_path
@@ -282,7 +308,7 @@ module Filbunke
             msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
             @logger.error "Failed to move hdfs file #{file.url}: #{msg}"
             return false
-          end  
+          end
         else
           @logger.error "Failed to update hdfs file #{file.url}! Unable to execute #{hdfs_cmd}"
           return false
@@ -295,8 +321,7 @@ module Filbunke
     end
 
     def write_file!(file_path, contents)
-      ::FileUtils.mkdir_p(::File.dirname(file_path))
-      @logger.debug("Updating: #{file_path}")
+
       begin
         ::File.open("#{file_path}.tmp", 'w') do |file|
           file.write(contents)
@@ -308,7 +333,7 @@ module Filbunke
         msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
         @logger.error "Failed to move file #{file_path}: #{msg}"
         return false
-      end  
+      end
     end
 
     def delete_file!(file_path)

data/lib/filbunke/repository.rb

@@ -1,18 +1,18 @@
 module Filbunke
   class Repository
-    attr_accessor :name, 
-                  :host, 
-                  :port, 
-                  :local_path, 
-                  :file_umask, 
-                  :directory_umask, 
-                  :user, 
-                  :pass, 
-                  :hadoop_binary, 
+    attr_accessor :name,
+                  :host,
+                  :port,
+                  :local_path,
+                  :file_umask,
+                  :directory_umask,
+                  :user,
+                  :pass,
+                  :hadoop_binary,
                   :run_every,
                   :hydra_concurrency,
                   :batch_size
- 
+
     def initialize(repository_config)
       @name = repository_config["filbunke_server_repository"]
       @host = repository_config["filbunke_server_host"]
@@ -28,5 +28,5 @@ module Filbunke
       # batch_size == 0 means use default configured in filbunke-server
       @batch_size = repository_config.fetch("batch_size", 0).to_i
    end
-
-end
+ end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filbunke
 version: !ruby/object:Gem::Version
-  version: 1.12.0
+  version: 1.13.0
 platform: ruby
 authors:
 - Wouter de Bie
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-04 00:00:00.000000000 Z
+date: 2016-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thoughtbot-shoulda
@@ -47,14 +47,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: open4
   requirement: !ruby/object:Gem::Requirement