filbunke 1.13.5 → 2.0.2

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ZTY5MmMwZWFjM2U3YThjYzA1ZjkzMmRjNjVmNzVkZDI4Yzk0N2EzZA==
-  data.tar.gz: !binary |-
-    NzlmYmMzN2YwYjI1ZTA5MzhkNThkNWQ0ZDk4YTZiMmQxMWZhNGY4OA==
+SHA1:
+  metadata.gz: 9ebdbdfb5aa1fc9a53ffe60b2b79d8cec6fe920c
+  data.tar.gz: d320cceb5cd74e3c443495fe2305e08a7d385af8
 SHA512:
-  metadata.gz: !binary |-
-    MGY2Mjg0OWM4NDk1YTRjNDMzNDhhOGJlMTA1ZGI2NjAyM2Q0MzA3MDNlMjY0
-    YzkzMzAxOTc2ZmUxMGZlYmRhM2VhYTg1MGI2ZDY4YzAyMmFjNTY1OWNmZDMx
-    ODczMzhhZjVhOWRmYWM5YTMxZTVlOTdkZTRhNmIwOWVkNjJhYzc=
-  data.tar.gz: !binary |-
-    YTg2Y2NmNzdjYjAxZTRlOTg2NDdlMGIxMTUyOTUwZDdlOGY1ZGRlY2U0NDM2
-    NTljMTc4ZGFlYjIzNWFkMDFmNzJiZmFiYTJmNTY5YzFlZWRkYzNkMmZkNjQ2
-    YWMzYTc0MjY3OTk2ZjkyYTQ4YzAzMDhmOGM3ZTU2OTVjNjBkM2E=
+  metadata.gz: dbd84ac518738229f3333557d68fa8c39dbe9153358ba2da521b575b2003bc814d595bc0d8bc256ddc3dee8965b7eefdb8c72117e0fd244d7847d425126e9fcb
+  data.tar.gz: 815b0982c1c9fab0fdb5c08fd34140796bd89e8cf1cf4c9ed00d474a71dec52ca301f88d60105038ceced0a41d31346a589a95c048ec79cc62c7dc8d01c25990

data/.ruby-version

@@ -1 +1 @@
-1.9.3
+2.2

data/Rakefile

@@ -13,10 +13,11 @@ begin
     gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     gem.files.exclude 'pkg'
     gem.executables = ['filbunked']
-    gem.add_dependency 'json', '= 1.7.5'
-    gem.add_dependency 'typhoeus', '= 1.0.1'
-    gem.add_dependency 'open4', '= 1.3.0'
-    gem.add_dependency 'mime-types', '= 1.19'
+    gem.add_dependency 'json', '= 1.8.3'
+    gem.add_dependency 'typhoeus', '= 0.7.3'
+    gem.add_dependency 'open4', '= 1.3.4'
+    gem.add_dependency 'mime-types', '= 2.6.2'
+    gem.add_dependency 'parallel', '= 1.6.1'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-1.13.5
+2.0.2

data/filbunke.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: filbunke 1.13.5 ruby lib
+# stub: filbunke 2.0.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "filbunke"
-  s.version = "1.13.5"
+  s.version = "2.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
-  s.date = "2016-03-11"
+  s.date = "2015-09-19"
   s.description = "Filbunke client and library"
   s.email = "technical@deltaprojects.com"
   s.executables = ["filbunked"]
@@ -38,12 +38,11 @@ Gem::Specification.new do |s|
     "lib/filbunke/file.rb",
     "lib/filbunke/logger.rb",
     "lib/filbunke/repository.rb",
-    "lib/filbunke/thread_pool.rb",
     "test/helper.rb",
     "test/test_filbunke.rb"
   ]
   s.homepage = "https://rubygems.org/gems/filbunke"
-  s.rubygems_version = "2.4.8"
+  s.rubygems_version = "2.4.5.1"
   s.summary = "Filbunke client"
 
   if s.respond_to? :specification_version then
@@ -51,23 +50,26 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
-      s.add_runtime_dependency(%q<json>, ["= 1.7.5"])
-      s.add_runtime_dependency(%q<typhoeus>, ["= 1.0.1"])
-      s.add_runtime_dependency(%q<open4>, ["= 1.3.0"])
-      s.add_runtime_dependency(%q<mime-types>, ["= 1.19"])
+      s.add_runtime_dependency(%q<json>, ["= 1.8.3"])
+      s.add_runtime_dependency(%q<typhoeus>, ["= 0.7.3"])
+      s.add_runtime_dependency(%q<open4>, ["= 1.3.4"])
+      s.add_runtime_dependency(%q<mime-types>, ["= 2.6.2"])
+      s.add_runtime_dependency(%q<parallel>, ["= 1.6.1"])
     else
       s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
-      s.add_dependency(%q<json>, ["= 1.7.5"])
-      s.add_dependency(%q<typhoeus>, ["= 1.0.1"])
-      s.add_dependency(%q<open4>, ["= 1.3.0"])
-      s.add_dependency(%q<mime-types>, ["= 1.19"])
+      s.add_dependency(%q<json>, ["= 1.8.3"])
+      s.add_dependency(%q<typhoeus>, ["= 0.7.3"])
+      s.add_dependency(%q<open4>, ["= 1.3.4"])
+      s.add_dependency(%q<mime-types>, ["= 2.6.2"])
+      s.add_dependency(%q<parallel>, ["= 1.6.1"])
     end
   else
     s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
-    s.add_dependency(%q<json>, ["= 1.7.5"])
-    s.add_dependency(%q<typhoeus>, ["= 1.0.1"])
-    s.add_dependency(%q<open4>, ["= 1.3.0"])
-    s.add_dependency(%q<mime-types>, ["= 1.19"])
+    s.add_dependency(%q<json>, ["= 1.8.3"])
+    s.add_dependency(%q<typhoeus>, ["= 0.7.3"])
+    s.add_dependency(%q<open4>, ["= 1.3.4"])
+    s.add_dependency(%q<mime-types>, ["= 2.6.2"])
+    s.add_dependency(%q<parallel>, ["= 1.6.1"])
   end
 end
 

data/lib/filbunke/callbacks.rb

@@ -6,27 +6,12 @@ module Filbunke
       @logger = logger
     end
 
-    def on_update_batch(files)
-      files.each do |item|
-        on_update(item.file, item.local_file_path)
-      end
-    end
     def on_update(file, local_file_path)
     end
 
-    def on_no_change_batch(files)
-      files.each do |item|
-        on_no_change(item.file, item.local_file_path)
-      end
-    end
     def on_no_change(file, local_file_path)
     end
-
-    def on_delete_batch(files)
-      files.each do |item|
-        on_delete(item.file, item.local_file_path)
-      end
-    end
+    
     def on_delete(file, local_file_path)
     end
 

data/lib/filbunke/client.rb

@@ -16,6 +16,7 @@ module Filbunke
       @repository = repository
       @logger = logger
       @callbacks = callbacks
+      @parallel_callback_opts = (@repository.num_callback_threads > 0 ? {:in_threads => @repository.num_callback_threads} : {:in_processes => repository.num_callback_processes} )
       @failed_request_log_file_name = failed_request_log_file_name
       @hydra = Typhoeus::Hydra.new(:max_concurrency => @repository.hydra_concurrency)
 
@@ -25,26 +26,18 @@ module Filbunke
     def with_updated_files(last_checkpoint)
       updates = get_updated_file_list(last_checkpoint)
       updated_files = updates["files"] || []
+      failure = false
+      
       new_checkpoint = updates["checkpoint"]
-      if updated_files.empty?
-        if updates.key?("files")
-          return begin
-            fetch_remote_last_checkpoint
-          rescue => e
-            @logger.warn "Failed to fetch remote last_checkpoint #{@repository.name} will fall back to local last_checkpoint=#{last_checkpoint}"
-            last_checkpoint
-          end
-        else
-          return last_checkpoint
-        end
-      end
+      
+      @logger.info "Updating repository: #{@repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}" if updated_files.size > 0
 
-      @logger.info "Updating repository: #{@repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}"
       @async_requests = []
+      
       callbacks_on_update = []
       callbacks_on_no_change = []
       callbacks_on_delete = []
-      has_update_file_failure = false
+
       updated_files.each do |raw_file|
         file = File.new(raw_file)
         local_file_path = ::File.join(@repository.local_path, file.path)
@@ -58,56 +51,41 @@ module Filbunke
               callbacks_on_update << OpenStruct.new({ :file => file, :local_file_path => local_file_path })
             else
               @logger.error "Unable to get file #{file.url} ==> #{file.path}!"
-              has_update_file_failure = true
-              break
+              failure = true
             end
+            
           else
             @logger.debug "File exists with correct hash: #{local_file_path}"
             callbacks_on_no_change << OpenStruct.new({:file => file, :local_file_path => local_file_path})
           end
         end
       end
+      @hydra.run
 
-      if has_update_file_failure
-        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
-        return last_checkpoint
-      end
-      @logger.info "Done setting up async requests for #{@repository.name},  starting fetch..."
-      has_fetch_failures = begin
-        @hydra.run
-        # Magnus 20160305 - since we now fail fast by raising a RuntimeError on response.code != 200
-        # I think we can remove the following request validation
-        @async_requests.any? do |request|
-          @logger.warn "request did not handle response: #{request.inspect}" if request.response.nil? || request.response.code != 200
-          request.response.nil? || request.response.code != 200
-        end
-      rescue RuntimeError, SystemCallError, StandardError => e
-        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
-        true
-      end
-
-      if has_fetch_failures
-        @logger.error "FAILED to fetch files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
-        return last_checkpoint
+      pfailure = failure || @async_requests.any? do |request| 
+        @logger.warn "request did not handle response: #{request.inspect}" if request.response.nil? || request.response.code != 200
+        request.response.nil? || request.response.code != 200
       end
 
-      @logger.info "Done fetching files for #{@repository.name}, processing callbacks..."
-      new_or_last_checkpoint = begin
-        run_callbacks_delete(callbacks_on_delete)
-        run_callbacks(callbacks_on_update)
-        run_callbacks_no_change(callbacks_on_no_change)
-
-        new_checkpoint || last_checkpoint
-      rescue RuntimeError, SystemCallError, StandardError => e
-        msg = ["Callbacks failed to run; #{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-        @logger.error "FAILED to process callbacks for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
+      if pfailure == false
+        @logger.info "Done fetching files for #{@repository.name}, processing callbacks..."
+        begin
+          run_callbacks_delete(callbacks_on_delete)
+          run_callbacks(callbacks_on_update)
+          run_callbacks_no_change(callbacks_on_no_change)
+
+          new_checkpoint || last_checkpoint
+        rescue RuntimeError, SystemCallError, StandardError => e
+          msg = ["Callbacks failed to run; #{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+          @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
+          last_checkpoint
+        end
+      else
+        @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
         last_checkpoint
       end
-
-      new_or_last_checkpoint
     end
-
+    
     def update_files!(last_checkpoint)
       with_updated_files(last_checkpoint) {}
     end
@@ -159,7 +137,7 @@ module Filbunke
       end
     end
 
-    def fetch_remote_last_checkpoint
+    def last_checkpoint
      last_checkpoint_http = Net::HTTP.new(@repository.host, @repository.port)
       last_checkpoint_http.start do |http|
         last_checkpoint_path = "/#{UPDATES_ACTION}/#{@repository.name}/#{LAST_CHECKPOINT_ACTION}"
@@ -171,7 +149,7 @@ module Filbunke
         return response.body.chomp.to_i
       end
     end
-
+    
     private
 
     def log_failed_request(failed_request_command, e)
@@ -183,11 +161,11 @@ module Filbunke
     end
 
     def update_file!(file, local_file_path)
-
+        
       if file.url =~ /^http:\/\//
         update_http_file!(file, local_file_path)
       elsif (file.url =~ /^hdfs:\/\//)
-        update_hdfs_file!(file, local_file_path)
+        success = update_hdfs_file!(file, local_file_path)
       else
         raise "Unsupported protocol for file: #{file.inspect}"
       end
@@ -195,19 +173,25 @@ module Filbunke
 
     def run_callbacks(files)
       @callbacks.each do |callback|
-        callback.on_update_batch(files)
+        Parallel.map(files, @parallel_callback_opts) do |item|
+          callback.on_update(item.file, item.local_file_path)  
+        end
       end
     end
 
     def run_callbacks_no_change(files)
       @callbacks.each do |callback|
-        callback.on_no_change_batch(files)
+        Parallel.map(files, @parallel_callback_opts) do |item|
+          callback.on_no_change(item.file, item.local_file_path)
+        end
       end
     end
 
     def run_callbacks_delete(files)
       @callbacks.each do |callback|
-        callback.on_delete_batch(files)
+        Parallel.map(files, @parallel_callback_opts) do |item|
+          callback.on_delete(item.file, item.local_file_path)
+        end
       end
     end
 
@@ -226,7 +210,6 @@ module Filbunke
         updates_http.read_timeout = 300 # default is 60 seconds
         updates_http.start do |http|
           updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}"
-          updates_path = "#{updates_path}&batch_size=#{@repository.batch_size}" if @repository.batch_size > 0
           begin
             @logger.info "Fetching updated file list from #{updates_path}"
             request = Net::HTTP::Get.new(updates_path)
@@ -245,67 +228,44 @@ module Filbunke
       rescue StandardError => e
         @logger.error "Unable to create HTTP connection to #{@repository.host}:#{@repository.port} (#{e.message})!"
         return {}
-      end
+      end        
     end
 
     def update_http_file!(file, local_file_path)
-      downloaded_file = nil
-      tmp_filename = "#{local_file_path}.tmp"
       begin
         async_request = if @repository.user
-          Typhoeus::Request.new(
-            URI.escape(file.url),
-            :followlocation => true,
-            :username => @repository.user,
-            :password => @repository.pass
-          )
+          Typhoeus::Request.new(URI.encode(file.url), :followlocation => true, :username => @repository.user, :password => @repository.pass)
         else
-          Typhoeus::Request.new(
-            URI.escape(file.url),
-            :followlocation => true
-          )
-        end
-
-        
-        async_request.on_headers do |response|
-          if response.code != 200
-            raise "Failed to fetch response(#{response.code}) for url '#{response.effective_url}' ---\n\t #{response.inspect}"
-          end
-          @logger.debug("Updating: #{local_file_path}")
-          ::FileUtils.mkdir_p(::File.dirname(local_file_path))
-          downloaded_file = ::File.new(tmp_filename, "wb")
-        end
-
-        async_request.on_body do |chunk, response|
-          downloaded_file.write(chunk) if response.code == 200
+          Typhoeus::Request.new(URI.encode(file.url), :followlocation => true)
         end
-
         async_request.on_complete do |response|
-          unless downloaded_file.nil?
-            downloaded_file.close
-          end
-          if ::File.exist?(tmp_filename)
-            if response.code == 200
-              ::FileUtils.mv(tmp_filename, local_file_path) 
+          success = false
+          begin
+            success = response.code.to_i == 200
+            if success
+              write_file!(local_file_path, response.body)
             else
-              ::FileUtils.rm(tmp_filename)
+              body_if_error = response.code >= 500 ? ", body = #{response.body}" : ""
+              @logger.warn "Failed to update file #{file.url}, got status code = #{response.code}#{body_if_error}"
             end
+          rescue SystemCallError, StandardError => e
+            msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+            @logger.error "Failed to update file #{file.url}: #{msg}"
           end
-          true
+          # return the async_request.handled_response value here
+          success
         end
         @hydra.queue async_request
         @async_requests << async_request
-        true
-      rescue RuntimeError, SystemCallError, StandardError => e
+      rescue StandardError => e
         msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
         @logger.error "Failed to update file #{file.url}: #{msg}"
-        if ::File.exist?(tmp_filename)
-          ::FileUtils.rm(tmp_filename)
-        end
-        false
+        return false
       end
-    end
 
+      return true
+    end
+    
     def update_hdfs_file!(file, local_file_path)
       begin
         ::FileUtils.mkdir_p(::File.dirname(local_file_path))
@@ -314,10 +274,10 @@ module Filbunke
         url.gsub!(/hdfs:\/\/([^\/]*)(.*)/, "hdfs://\\2")
         hdfs_cmd = "#{@repository.hadoop_binary} dfs -copyToLocal #{url} #{local_file_path}.tmp"
         #@logger.debug "Trying to update #{local_file_path} with '#{hdfs_cmd}'"
-
+        
         pid, stdin, stdout, stderr = Open4::popen4 hdfs_cmd
         ignored, status = Process::waitpid2 pid
-
+              
         if status.exitstatus == 0 then
           begin
             ::FileUtils.mv "#{local_file_path}.tmp", local_file_path
@@ -326,7 +286,7 @@ module Filbunke
             msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
             @logger.error "Failed to move hdfs file #{file.url}: #{msg}"
             return false
-          end
+          end  
         else
           @logger.error "Failed to update hdfs file #{file.url}! Unable to execute #{hdfs_cmd}"
           return false
@@ -338,6 +298,23 @@ module Filbunke
       end
     end
 
+    def write_file!(file_path, contents)
+      ::FileUtils.mkdir_p(::File.dirname(file_path))
+      @logger.debug("Updating: #{file_path}")
+      begin
+        ::File.open("#{file_path}.tmp", 'w') do |file|
+          file.write(contents)
+          file.close
+        end
+        ::FileUtils.mv "#{file_path}.tmp", file_path
+        return true
+      rescue StandardError => e
+        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "Failed to move file #{file_path}: #{msg}"
+        return false
+      end  
+    end
+
     def delete_file!(file_path)
       if ::File.exists?(file_path) then
         @logger.debug("Deleting: #{file_path}")

data/lib/filbunke/daemon.rb

@@ -15,7 +15,6 @@ module Filbunke
         @logger.log("Initializing repository: #{repository_name}")
         @clients << begin
           repository_config["run_every"] = repository_config.fetch("run_every", @config.fetch("run_every", 10))
-          repository_config["batch_size"] = repository_config.fetch("batch_size", @config.fetch("batch_size", 0))
           repository = Repository.new(repository_config)
           callbacks = []
           repository_config["callbacks"].each do |callback_name, callback_config|
@@ -26,10 +25,6 @@ module Filbunke
           failed_request_log_file_name = repository_config["failed_request_log_file_name"]||nil
           
           Client.new(repository, @logger, callbacks, failed_request_log_file_name)
-        rescue => e
-          msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
-          @logger.error("Failed to initialize #{repository_name}; #{msg}")
-          raise e
         end
       end
     end
@@ -48,13 +43,14 @@ module Filbunke
               update_checkpoint_for_repository(client.repository, new_checkpoint)
               sleep client.repository.run_every
             end
-          rescue => e
+          rescue RuntimeError, SystemCallError, StandardError => e
             msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
             @logger.error("#{client.repository.name} Died..  #{msg}")
             Process.kill("KILL", @parent_pid)
             exit 1
           end
         end
+      
       end
       client_pids.each { |pid| Process.wait(pid) }
     end
@@ -83,18 +79,6 @@ module Filbunke
     end
 
     def write_pid!(pid_file_path)
-        
-      begin
-        existing_process = ::File.read(pid_file_path).to_i if ::File.readable?(pid_file_path)
-        if existing_process != nil and existing_process > 0
-          @logger.info("killing existing process #{existing_process} from #{pid_file_path}")
-          Process.kill("KILL", existing_process)
-          
-        end
-      rescue => e
-        @logger.warn("failed to kill existing pid from #{pid_file_path}: #{e}\n\twill ignore and continue...")
-      end
-    
       ::File.open(pid_file_path, 'w') do |f|
         f.write(Process.pid.to_i)
         f.close

data/lib/filbunke/logger.rb

@@ -4,41 +4,41 @@ module Filbunke
 
     def initialize(log_file_name, local, level)
       @local = local
-      @log = if @local or log_file_name.nil?
+      @logger = if @local or log_file_name.nil?
         Logger.new(STDOUT) 
       else 
         Logger.new(log_file_name)
       end
 
-      @log.level = parse_level(level)
+      @logger.level = parse_level(level)
     end
 
     def puts(msg)
-      info(msg)
+      @logger.info(msg)
     end
-    
+
     def log(msg)
-      @log.info msg
+      @logger.info msg
     end
 
     def info(msg)
-      @log.info msg
+      @logger.info msg
     end
 
     def error(msg)
-      @log.error msg
+      @logger.error msg
     end
 
     def warn(msg)
-      @log.warn msg
+      @logger.warn msg
     end
 
     def debug(msg)
-      @log.debug msg
+      @logger.debug msg
     end
 
     def fatal(msg)
-      @log.error msg
+      @logger.error msg
     end
 
     def parse_level(constantOrString)

data/lib/filbunke/repository.rb

@@ -1,17 +1,19 @@
 module Filbunke
   class Repository
-    attr_accessor :name,
-                  :host,
-                  :port,
-                  :local_path,
-                  :file_umask,
-                  :directory_umask,
-                  :user,
-                  :pass,
-                  :hadoop_binary,
+
+    attr_accessor :name, 
+                  :host, 
+                  :port, 
+                  :local_path, 
+                  :file_umask, 
+                  :directory_umask, 
+                  :user, 
+                  :pass, 
+                  :hadoop_binary, 
                   :run_every,
                   :hydra_concurrency,
-                  :batch_size
+                  :num_callback_processes,
+                  :num_callback_threads
 
     def initialize(repository_config)
       @name = repository_config["filbunke_server_repository"]
@@ -25,8 +27,19 @@ module Filbunke
       @hadoop_binary = repository_config["hadoop_binary"]
       @run_every = repository_config.fetch("run_every", 10).to_i
       @hydra_concurrency = repository_config.fetch("hydra_concurrency", 100).to_i
-      # batch_size == 0 means use default configured in filbunke-server
-      @batch_size = repository_config.fetch("batch_size", 0).to_i
-   end
- end
-end
+      @num_callback_processes = repository_config["num_callback_processes"].to_i
+      @num_callback_threads = repository_config["num_callback_threads"].to_i
+
+      if @num_callback_threads == 0 and @num_callback_processes == 0
+        @num_callback_threads = Parallel.processor_count
+      end
+      
+      raise ConfigurationError.new("callbacks cant use both processes and threads  (#{@name})") if @num_callback_threads > 0 && @num_callback_processes > 0
+      
+    end
+
+  end
+
+  class ConfigurationError < StandardError
+  end
+end

data/lib/filbunke.rb

@@ -7,8 +7,8 @@ require 'typhoeus'
 require 'open4'
 require 'uri'
 require 'logger'
+require 'parallel'
 
-require File.expand_path(File.dirname(__FILE__) + '/filbunke/thread_pool.rb')
 require File.expand_path(File.dirname(__FILE__) + '/filbunke/client.rb')
 require File.expand_path(File.dirname(__FILE__) + '/filbunke/file.rb')
 require File.expand_path(File.dirname(__FILE__) + '/filbunke/repository.rb')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filbunke
 version: !ruby/object:Gem::Version
-  version: 1.13.5
+  version: 2.0.2
 platform: ruby
 authors:
 - Wouter de Bie
@@ -11,20 +11,20 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-11 00:00:00.000000000 Z
+date: 2015-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thoughtbot-shoulda
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -33,56 +33,70 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.5
+        version: 1.8.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.5
+        version: 1.8.3
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 0.7.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 0.7.3
 - !ruby/object:Gem::Dependency
   name: open4
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.3.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.3.4
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: 2.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: 2.6.2
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.1
 description: Filbunke client and library
 email: technical@deltaprojects.com
 executables:
@@ -90,9 +104,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .ruby-gemset
-- .ruby-version
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
 - Gemfile
 - LICENSE
 - README.rdoc
@@ -112,7 +126,6 @@ files:
 - lib/filbunke/file.rb
 - lib/filbunke/logger.rb
 - lib/filbunke/repository.rb
-- lib/filbunke/thread_pool.rb
 - test/helper.rb
 - test/test_filbunke.rb
 homepage: https://rubygems.org/gems/filbunke
@@ -124,17 +137,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Filbunke client

data/lib/filbunke/thread_pool.rb

@@ -1,104 +0,0 @@
-# Inspired by https://github.com/meh/ruby-threadpool
-require 'thread'
-
-class ThreadPool
-
-  class Job < Struct.new(:args, :block); end
-
-  def initialize(min, max = nil)
-    
-    trap("INT") { shutdown }
-
-    @min = min
-    @max = max || min
-
-    @cv = ConditionVariable.new
-    @mutex = Mutex.new
-
-    @queue = []
-    @workers = []
-
-    @spawned = 0
-    @waiting = 0
-    @shutdown = false
-    @queue_locked = false
-
-    @mutex.synchronize do
-      min.times { spawn_thread }
-    end
-  end
-
-  def execute(*args, &block)
-    @mutex.synchronize do
-      raise "Thread pool is about to shutdown" if @shutdown || @queue_locked
-
-      @queue << Job.new(args, block)
-
-      spawn_thread if @waiting == 0 && @spawned < @max
-
-      @cv.signal
-    end
-  end
-  alias :<< :execute
-
-  def shutdown
-    @mutex.synchronize do
-      @shutdown = true
-      @cv.broadcast
-    end
-
-    @workers.first.join until @workers.empty?
-  end
-
-  def join
-    @mutex.synchronize do
-      @queue_locked = true
-      @cv.broadcast
-      sleep 0.01 until @queue.empty?
-    end
-    shutdown
-  end
-
-  protected
-
-  def spawn_thread
-    thread = Thread.new do
-      continue = true
-
-      while continue do
-        job = nil
-
-        @mutex.synchronize do
-          while @queue.empty? && continue
-            if @shutdown || @queue_locked
-              continue = false
-              break
-            end
-
-            @waiting += 1
-            @cv.wait @mutex
-            @waiting -= 1
-
-            if @shutdown || @queue_locked
-              continue = false
-              break
-            end
-          end
-
-          if continue
-            job = @queue.shift
-            job.block.call(*job.args) if job
-          end
-        end
-      end
-
-      @mutex.synchronize do
-        @spawned -= 1
-        @workers.delete thread
-      end
-    end
-
-    @workers << thread
-    thread
-  end
-end