filbunke 2.0.9 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c42ae52d1bc54da6a935b95b029a6bf1797d6d34
-  data.tar.gz: 1435482387aa797f0a0780aa65d79a3ecacd17a3
+  metadata.gz: bdf27520bab96738e5a12465ccd526ee4d3d837a
+  data.tar.gz: dbb247721d46a2b8aafe2d5942120850b304fd3a
 SHA512:
-  metadata.gz: 4b268216e4e4db244c77d734f935507061631f4ac568ce6e9009e991837af4dd25fd9dd1bd205895c438c9380005b10b1c7c9aa6694e49ad63bbe3b15743762f
-  data.tar.gz: fbae049f4a7e51a0635348d1923fa6b0bb7b46bd672f982317614fdd51c8e96f882922cb28eebca12300c8b036d4dd78f7458cb5491b426a80430241abbf9f9b
+  metadata.gz: 6754b1ac1376a1f491d7159fe09ab95b474637b144d2ab09fdbea87fd724234e2cf9ee5d535a6678cfbcca585dd681f186df7c193cceb6e0480d3f154536825c
+  data.tar.gz: 17084f8cc91464c6a9a2d1a47f5b2df0c732bc338bbd877f2795af9e16d44c80c1aea2d0d38304adee1bd5e8c06dce0d8503518f720da45253e4dea1123c3ee9

data/.gitignore

@@ -1 +1,2 @@
-Gemfile.lock
+Gemfile.lock
+pkg/

data/Gemfile

@@ -1,3 +1,7 @@
 source 'http://rubygems.org'
 
-gem 'jeweler'
+
+group :test do
+  gem 'shoulda'
+  gem 'test-unit'
+end

data/Rakefile

@@ -3,6 +3,13 @@ require 'rake'
 
 begin
   require 'jeweler'
+  required_dependencies = {
+    "json" => "1.8.3",
+    "typhoeus" => "1.0.1",
+    "open4" => "1.3.4",
+    "mime-types" => "2.6.2",
+    "parallel" => "1.6.1"
+  }
   Jeweler::Tasks.new do |gem|
     gem.name = "filbunke"
     gem.summary = %Q{Filbunke client}
@@ -10,29 +17,33 @@ begin
     gem.email = "technical@deltaprojects.com"
     gem.homepage = "https://rubygems.org/gems/filbunke"
     gem.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
-    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.add_development_dependency "shoulda", "~> 0"
     gem.files.exclude 'pkg'
     gem.executables = ['filbunked']
-    gem.add_dependency 'json', '= 1.8.3'
-    gem.add_dependency 'typhoeus', '= 0.7.3'
-    gem.add_dependency 'open4', '= 1.3.4'
-    gem.add_dependency 'mime-types', '= 2.6.2'
-    gem.add_dependency 'parallel', '= 1.6.1'
+    required_dependencies.each do |name, version|
+      gem.add_dependency "#{name}", "= #{version}"
+    end
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError => e
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+  puts "Jeweler (or a dependency) not available. Install it with: \n\tgem install jeweler \nadditional dependencies;"
+  required_dependencies.each do |name,version|
+    puts "\tgem install #{name} -v #{version}"
+  end
   raise e
 end
 
+
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  test.warning = true
 end
 
+
 begin
   require 'rcov/rcovtask'
   Rcov::RcovTask.new do |test|
@@ -46,6 +57,10 @@ rescue LoadError
   end
 end
 
+task :check_dependencies do
+
+end
+
 task :test => :check_dependencies
 
 task :default => :test

data/VERSION

@@ -1 +1 @@
-2.0.9
+2.1.0

data/filbunke.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: filbunke 2.0.9 ruby lib
+# stub: filbunke 2.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "filbunke"
-  s.version = "2.0.9"
+  s.version = "2.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
-  s.date = "2015-11-05"
+  s.date = "2016-03-05"
   s.description = "Filbunke client and library"
   s.email = "technical@deltaprojects.com"
   s.executables = ["filbunked"]
@@ -42,31 +42,31 @@ Gem::Specification.new do |s|
     "test/test_filbunke.rb"
   ]
   s.homepage = "https://rubygems.org/gems/filbunke"
-  s.rubygems_version = "2.5.0"
+  s.rubygems_version = "2.4.5.1"
   s.summary = "Filbunke client"
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, ["~> 0"])
       s.add_runtime_dependency(%q<json>, ["= 1.8.3"])
-      s.add_runtime_dependency(%q<typhoeus>, ["= 0.7.3"])
+      s.add_runtime_dependency(%q<typhoeus>, ["= 1.0.1"])
       s.add_runtime_dependency(%q<open4>, ["= 1.3.4"])
       s.add_runtime_dependency(%q<mime-types>, ["= 2.6.2"])
       s.add_runtime_dependency(%q<parallel>, ["= 1.6.1"])
     else
-      s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_dependency(%q<shoulda>, ["~> 0"])
       s.add_dependency(%q<json>, ["= 1.8.3"])
-      s.add_dependency(%q<typhoeus>, ["= 0.7.3"])
+      s.add_dependency(%q<typhoeus>, ["= 1.0.1"])
       s.add_dependency(%q<open4>, ["= 1.3.4"])
       s.add_dependency(%q<mime-types>, ["= 2.6.2"])
       s.add_dependency(%q<parallel>, ["= 1.6.1"])
     end
   else
-    s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    s.add_dependency(%q<shoulda>, ["~> 0"])
     s.add_dependency(%q<json>, ["= 1.8.3"])
-    s.add_dependency(%q<typhoeus>, ["= 0.7.3"])
+    s.add_dependency(%q<typhoeus>, ["= 1.0.1"])
     s.add_dependency(%q<open4>, ["= 1.3.4"])
     s.add_dependency(%q<mime-types>, ["= 2.6.2"])
     s.add_dependency(%q<parallel>, ["= 1.6.1"])

data/lib/filbunke/client.rb

@@ -10,7 +10,6 @@ module Filbunke
     URL_KEY = 'url'
     FROM_CHECKPOINT_KEY = 'from_checkpoint'
     HASH_KEY = 'hash'
-    URI_UNSAFE_CHARACTERS = '/[^.:\/\w-]/'
 
 
     def initialize(repository, logger, callbacks = [], failed_request_log_file_name = nil)
@@ -26,18 +25,17 @@ module Filbunke
     def with_updated_files(last_checkpoint)
       updates = get_updated_file_list(last_checkpoint)
       updated_files = updates["files"] || []
-      failure = false
-      
-      new_checkpoint = updates["checkpoint"]
-      
-      @logger.info "Updating repository: #{@repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}" if updated_files.size > 0
+      new_checkpoint = updates["checkpoint"] || 0
+      if updated_files.empty?
+        return new_checkpoint
+      end
+      @logger.info "Updating repository: #{@repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}"
 
       @async_requests = []
-      
       callbacks_on_update = []
       callbacks_on_no_change = []
       callbacks_on_delete = []
-
+      has_update_file_failure = false
       updated_files.each do |raw_file|
         file = File.new(raw_file)
         local_file_path = ::File.join(@repository.local_path, file.path)
@@ -50,42 +48,57 @@ module Filbunke
               yield file
               callbacks_on_update << OpenStruct.new({ :file => file, :local_file_path => local_file_path })
             else
-              @logger.error "Unable to get file #{file.url} ==> #{file.path}!"
-              failure = true
+              @logger.error "Unable to fetch file #{file.url} ==> #{file.path}!"
+              has_update_file_failure = true
+              break;
             end
-            
+
           else
             @logger.debug "File exists with correct hash: #{local_file_path}"
             callbacks_on_no_change << OpenStruct.new({:file => file, :local_file_path => local_file_path})
           end
         end
       end
-      @hydra.run
 
-      pfailure = failure || @async_requests.any? do |request| 
-        @logger.warn "request did not handle response: #{request.inspect}" if request.response.nil? || request.response.code != 200
-        request.response.nil? || request.response.code != 200
+      if has_update_file_failure
+        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
+        return last_checkpoint
       end
+      @logger.info "Done setting up async requests for #{@repository.name}, starting fetch..."
 
-      if pfailure == false
-        @logger.info "Done fetching files for #{@repository.name}, processing callbacks..."
-        begin
-          run_callbacks_delete(callbacks_on_delete)
-          run_callbacks(callbacks_on_update)
-          run_callbacks_no_change(callbacks_on_no_change)
-
-          new_checkpoint || last_checkpoint
-        rescue RuntimeError, SystemCallError, StandardError => e
-          msg = ["Callbacks failed to run; #{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-          @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
-          last_checkpoint
+      has_fetch_files_failure = begin
+        @hydra.run
+        @async_requests.any? do |request|
+          @logger.warn "request did not handle response: #{request.inspect}" if request.response.nil? || request.response.code != 200
+          request.response.nil? || request.response.code != 200
         end
-      else
+      rescue RuntimeError, SystemCallError, StandardError => e
+        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
+        true
+      end
+
+      if has_fetch_files_failure
         @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
+        return last_checkpoint
+      end
+
+      @logger.info "Done fetching files for #{@repository.name}, processing callbacks..."
+      new_or_last_checkpoint = begin
+        run_callbacks_delete(callbacks_on_delete)
+        run_callbacks(callbacks_on_update)
+        run_callbacks_no_change(callbacks_on_no_change)
+
+        new_checkpoint || last_checkpoint
+      rescue RuntimeError, SystemCallError, StandardError => e
+        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "FAILED to process callbacks for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
         last_checkpoint
       end
+
+      new_or_last_checkpoint
     end
-    
+
     def update_files!(last_checkpoint)
       with_updated_files(last_checkpoint) {}
     end
@@ -138,7 +151,7 @@ module Filbunke
     end
 
     def last_checkpoint
-     last_checkpoint_http = Net::HTTP.new(@repository.host, @repository.port)
+      last_checkpoint_http = Net::HTTP.new(@repository.host, @repository.port)
       last_checkpoint_http.start do |http|
         last_checkpoint_path = "/#{UPDATES_ACTION}/#{@repository.name}/#{LAST_CHECKPOINT_ACTION}"
         request = Net::HTTP::Get.new(last_checkpoint_path)
@@ -149,7 +162,7 @@ module Filbunke
         return response.body.chomp.to_i
       end
     end
-    
+
     private
 
     def log_failed_request(failed_request_command, e)
@@ -161,7 +174,7 @@ module Filbunke
     end
 
     def update_file!(file, local_file_path)
-        
+
       if file.url =~ /^http:\/\//
         update_http_file!(file, local_file_path)
       elsif (file.url =~ /^hdfs:\/\//)
@@ -204,6 +217,7 @@ module Filbunke
         updates_http.read_timeout = 300 # default is 60 seconds
         updates_http.start do |http|
           updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}"
+          updates_path = "#{updates_path}&batch_size=#{@repository.batch_size}" if @repository.batch_size > 0
           begin
             @logger.info "Fetching updated file list from #{updates_path}"
             request = Net::HTTP::Get.new(updates_path)
@@ -228,38 +242,59 @@ module Filbunke
     def update_http_file!(file, local_file_path)
       begin
         async_request = if @repository.user
-          Typhoeus::Request.new(URI.encode(file.url, URI_UNSAFE_CHARACTERS), :followlocation => true, :username => @repository.user, :password => @repository.pass)
+          Typhoeus::Request.new(
+            URI.escape(file.url),
+            :followlocation => true,
+            :username => @repository.user,
+            :password => @repository.pass
+          )
         else
-          Typhoeus::Request.new(URI.encode(file.url, URI_UNSAFE_CHARACTERS), :followlocation => true)
+          Typhoeus::Request.new(
+            URI.escape(file.url),
+            :followlocation => true
+          )
+        end
+
+        downloaded_file = nil
+        async_request.on_headers do |response|
+          if response.code != 200
+            raise "Downloading file #{response.effective_url} failed with status code #{response.code} --- #{response.inspect}"
+          end
+          ::FileUtils.mkdir_p(::File.dirname(local_file_path))
+          downloaded_file = ::File.new("#{local_file_path}.tmp", "wb")
+          @logger.debug("Updating file #{local_file_path}")
+        end
+
+
+        async_request.on_body do |chunk, response|
+          downloaded_file.write(chunk) if response.code == 200
         end
+
         async_request.on_complete do |response|
-          success = false
-          begin
-            success = response.code.to_i == 200
-            if success
-              write_file!(local_file_path, response.body)
+          unless downloaded_file.nil?
+            downloaded_file.close unless downloaded_file.closed?
+            if response.code == 200
+              ::FileUtils.mv "#{local_file_path}.tmp", local_file_path
             else
-              body_if_error = response.code >= 500 ? ", body = #{response.body}" : ""
-              @logger.warn "Failed to update file #{file.url}, got status code = #{response.code}#{body_if_error}"
+              ::FileUtils.rm "#{local_file_path}.tmp" if ::File.exist? "#{local_file_path}.tmp"
             end
-          rescue SystemCallError, StandardError => e
-            msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-            @logger.error "Failed to update file #{file.url}: #{msg}"
           end
-          # return the async_request.handled_response value here
-          success
+          true
         end
         @hydra.queue async_request
         @async_requests << async_request
-      rescue StandardError => e
+        true
+      rescue RuntimeError, SystemCallError, StandardError => e
         msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
         @logger.error "Failed to update file #{file.url}: #{msg}"
-        return false
+        unless downloaded_file.nil?
+          downloaded_file.close unless downloaded_file.closed?
+          ::FileUtils.rm "#{local_file_path}.tmp" if ::File.exist? "#{local_file_path}.tmp"
+        end
+        false
       end
-
-      return true
     end
-    
+
     def update_hdfs_file!(file, local_file_path)
       begin
         ::FileUtils.mkdir_p(::File.dirname(local_file_path))
@@ -268,10 +303,10 @@ module Filbunke
         url.gsub!(/hdfs:\/\/([^\/]*)(.*)/, "hdfs://\\2")
         hdfs_cmd = "#{@repository.hadoop_binary} dfs -copyToLocal #{url} #{local_file_path}.tmp"
         #@logger.debug "Trying to update #{local_file_path} with '#{hdfs_cmd}'"
-        
+
         pid, stdin, stdout, stderr = Open4::popen4 hdfs_cmd
         ignored, status = Process::waitpid2 pid
-              
+
         if status.exitstatus == 0 then
           begin
             ::FileUtils.mv "#{local_file_path}.tmp", local_file_path
@@ -280,7 +315,7 @@ module Filbunke
             msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
             @logger.error "Failed to move hdfs file #{file.url}: #{msg}"
             return false
-          end  
+          end
         else
           @logger.error "Failed to update hdfs file #{file.url}! Unable to execute #{hdfs_cmd}"
           return false
@@ -292,30 +327,11 @@ module Filbunke
       end
     end
 
-    def write_file!(file_path, contents)
-      ::FileUtils.mkdir_p(::File.dirname(file_path))
-      @logger.debug("Updating: #{file_path}")
-      begin
-        ::File.open("#{file_path}.tmp", 'w') do |file|
-          file.write(contents)
-          file.close
-        end
-        ::FileUtils.mv "#{file_path}.tmp", file_path
-        return true
-      rescue StandardError => e
-        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-        @logger.error "Failed to move file #{file_path}: #{msg}"
-        return false
-      end  
-    end
-
     def delete_file!(file_path)
       if ::File.exists?(file_path) then
         @logger.debug("Deleting: #{file_path}")
         ::File.delete(file_path)
       end
     end
-
   end
 end
-

data/lib/filbunke/daemon.rb

@@ -15,6 +15,7 @@ module Filbunke
         @logger.log("Initializing repository: #{repository_name}")
         @clients << begin
           repository_config["run_every"] = repository_config.fetch("run_every", @config.fetch("run_every", 10))
+          repository_config["batch_size"] = repository_config.fetch("batch_size", @config.fetch("batch_size", 0))
           repository = Repository.new(repository_config)
           callbacks = []
           repository_config["callbacks"].each do |callback_name, callback_config|

data/lib/filbunke/repository.rb

@@ -11,7 +11,8 @@ module Filbunke
                   :pass, 
                   :hadoop_binary, 
                   :run_every,
-                  :hydra_concurrency
+                  :hydra_concurrency,
+                  :batch_size
 
     def initialize(repository_config)
       @name = repository_config["filbunke_server_repository"]
@@ -25,6 +26,7 @@ module Filbunke
       @hadoop_binary = repository_config["hadoop_binary"]
       @run_every = repository_config.fetch("run_every", 10).to_i
       @hydra_concurrency = repository_config.fetch("hydra_concurrency", 100).to_i
+      @batch_size = repository_config.fetch("batch_size", 0).to_i
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filbunke
 version: !ruby/object:Gem::Version
-  version: 2.0.9
+  version: 2.1.0
 platform: ruby
 authors:
 - Wouter de Bie
@@ -11,20 +11,20 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-05 00:00:00.000000000 Z
+date: 2016-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: thoughtbot-shoulda
+  name: shoulda
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -47,14 +47,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.7.3
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.7.3
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: open4
   requirement: !ruby/object:Gem::Requirement
@@ -147,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.0
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Filbunke client