filbunke 1.10.3 → 1.11.6

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: c037040d9ad780d44b920b3a6dbe1df82463b250
-  data.tar.gz: 08d359f508446e24881402b5be25de5ea9897ce3
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    M2NkMTBmMzNhMTkwZjg1OWJjN2NkMzQyZWNjY2U2ZmVhMTU5YTY0Nw==
+  data.tar.gz: !binary |-
+    NzZkODc3ZTAxYWQxOTY4YjczOTdlZTM4ZTAzOTljY2YwZmQ0NWYzZA==
 SHA512:
-  metadata.gz: c1f3a1556b9a09dd8f334794636646abaf0814d3308ce1be1f47fe248ce94f99b3989bd5ab5226ff52f9dc9f35fa68d31250bb11f9b8fc57398f71730341effb
-  data.tar.gz: e58c070c818db39a1273699f13184ae208f292bde5f4c603e3f715aa1f1bbe19e69833e7e3dd952bf7acd1cac1988274c907b003e818faded4df04dc4e726ed0
+  metadata.gz: !binary |-
+    YWE5YTVhMzE0MTEzODU3OTZkMjdlZDgyODVjZTJhYmViNTY2OWUyYTJjM2E2
+    MTdmM2Y4YjY4OTM4MjNjZDE0YTVkMGVkOWI5ODM2NTcwYzczZDBlMzBmOWY0
+    Njg4NzRiODAxYWNmNzM2OTU0MmZiZmYwMDM4MGY0ZDExMzc4OWE=
+  data.tar.gz: !binary |-
+    YjA4NjZjZTcxMzVjZjkwY2MzNmI4MzQ0MDdlOWVkYmFkMmU3NjdmOGM1NjFm
+    NDA2MzlhMThhMDgxZjQwYWEwNDI2ZDM5MjZhMjg3MDFkMjFkODBhMWQ3NjY0
+    MWViMTgwN2NkYmQ1NDg1MDNjN2Q0NDdiZmJkOTc4YmEzZmU1ZDk=

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.ruby-gemset

@@ -0,0 +1 @@
+filbunkeclient

data/.ruby-version

@@ -0,0 +1 @@
+1.9.3

data/Gemfile

@@ -0,0 +1,3 @@
+source 'http://rubygems.org'
+
+gem 'jeweler'

data/Rakefile

@@ -7,9 +7,9 @@ begin
     gem.name = "filbunke"
     gem.summary = %Q{Filbunke client}
     gem.description = %Q{Filbunke client and library}
-    gem.email = "info@deltaprojects.com"
-    gem.homepage = "http://github.com/deltaprojects/filbunke"
-    gem.authors = ["Bjorn Sperber", "Karl Ravn"]
+    gem.email = "technical@deltaprojects.com"
+    gem.homepage = "https://rubygems.org/gems/filbunke"
+    gem.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
     gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     gem.files.exclude 'pkg'
     gem.executables = ['filbunked']
@@ -20,8 +20,9 @@ begin
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new
-rescue LoadError
+rescue LoadError => e
   puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+  raise e
 end
 
 require 'rake/testtask'

data/VERSION

@@ -1 +1 @@
-1.10.3
+1.11.6

data/doc/examples/filbunke_config.yml

@@ -2,6 +2,7 @@
 pid_file: '/var/run/filbunked.pid'
 user: 'wouter'
 log_file: '/tmp/filbunke.log'
+log_level: 'debug'
 checkpoint_path: '/var/tmp/filbunke_checkpoints/'
 run_every: 5
 callback_path: '/Users/wouter/prjs/filbunke-cdn/client/doc/examples'
@@ -16,6 +17,7 @@ repositories:
         file_umask: 0644
         directory_umask: 0755
         hadoop_binary: '/usr/bin/hadoop'
+        hydra_concurrency: 10
         callbacks:
           http_evict_cache:
             host: 'localhost'

data/filbunke.gemspec

@@ -2,18 +2,24 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: filbunke 1.11.6 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "filbunke"
-  s.version = "1.10.3"
+  s.version = "1.11.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Bjorn Sperber", "Karl Ravn"]
-  s.date = "2014-11-12"
+  s.require_paths = ["lib"]
+  s.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
+  s.date = "2015-09-22"
   s.description = "Filbunke client and library"
-  s.email = "info@deltaprojects.com"
+  s.email = "technical@deltaprojects.com"
   s.executables = ["filbunked"]
   s.files = [
+    ".gitignore",
+    ".ruby-gemset",
+    ".ruby-version",
+    "Gemfile",
     "LICENSE",
     "README.rdoc",
     "Rakefile",
@@ -32,12 +38,12 @@ Gem::Specification.new do |s|
     "lib/filbunke/file.rb",
     "lib/filbunke/logger.rb",
     "lib/filbunke/repository.rb",
+    "lib/filbunke/thread_pool.rb",
     "test/helper.rb",
     "test/test_filbunke.rb"
   ]
-  s.homepage = "http://github.com/deltaprojects/filbunke"
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.14"
+  s.homepage = "https://rubygems.org/gems/filbunke"
+  s.rubygems_version = "2.4.8"
   s.summary = "Filbunke client"
 
   if s.respond_to? :specification_version then

data/lib/filbunke.rb

@@ -6,7 +6,9 @@ require 'digest/md5'
 require 'typhoeus'
 require 'open4'
 require 'uri'
+require 'logger'
 
+require File.expand_path(File.dirname(__FILE__) + '/filbunke/thread_pool.rb')
 require File.expand_path(File.dirname(__FILE__) + '/filbunke/client.rb')
 require File.expand_path(File.dirname(__FILE__) + '/filbunke/file.rb')
 require File.expand_path(File.dirname(__FILE__) + '/filbunke/repository.rb')

data/lib/filbunke/callbacks.rb

@@ -6,12 +6,27 @@ module Filbunke
       @logger = logger
     end
 
+    def on_update_batch(files)
+      files.each do |item|
+        on_update(item.file, item.local_file_path)
+      end
+    end
     def on_update(file, local_file_path)
     end
 
+    def on_no_change_batch(files)
+      files.each do |item|
+        on_no_change(item.file, item.local_file_path)
+      end
+    end
     def on_no_change(file, local_file_path)
     end
-    
+
+    def on_delete_batch(files)
+      files.each do |item|
+        on_delete(item.file, item.local_file_path)
+      end
+    end
     def on_delete(file, local_file_path)
     end
 

data/lib/filbunke/client.rb

@@ -17,6 +17,9 @@ module Filbunke
       @logger = logger
       @callbacks = callbacks
       @failed_request_log_file_name = failed_request_log_file_name
+      @hydra = Typhoeus::Hydra.new(:max_concurrency => @repository.hydra_concurrency)
+
+      @logger.info "initialized client for repository '#{@repository.name}';  #{@repository.inspect}"
     end
 
     def with_updated_files(last_checkpoint)
@@ -26,41 +29,62 @@ module Filbunke
       
       new_checkpoint = updates["checkpoint"]
       
-      @logger.log "Updating repository: #{repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}" if updated_files.size > 0
+      @logger.info "Updating repository: #{@repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}" if updated_files.size > 0
 
-      @hydra = Typhoeus::Hydra.new(:max_concurrency => 10)
       @async_requests = []
+      
+      callbacks_on_update = []
+      callbacks_on_no_change = []
+      callbacks_on_delete = []
 
       updated_files.each do |raw_file|
         file = File.new(raw_file)
-
-        local_file_path = ::File.join(repository.local_path, file.path)
+        local_file_path = ::File.join(@repository.local_path, file.path)
         if file.state == "DELETED" then
           delete_file!(local_file_path)
-          run_callbacks_delete(file, local_file_path)
+          callbacks_on_delete << OpenStruct.new({ :file => file, :local_file_path => local_file_path })
         else
-        
           if file_needs_update?(file, local_file_path)
-          
             if update_file!(file, local_file_path) then
-
               yield file
+              callbacks_on_update << OpenStruct.new({ :file => file, :local_file_path => local_file_path })
             else
-              @logger.log "Unable to get file #{file.url} ==> #{file.path}!"
+              @logger.error "Unable to get file #{file.url} ==> #{file.path}!"
               failure = true
             end
-
           else
-            @logger.log "File exists with correct hash: #{local_file_path}"
-            run_callbacks_no_change(file, local_file_path)
+            @logger.debug "File exists with correct hash: #{local_file_path}"
+            callbacks_on_no_change << OpenStruct.new({:file => file, :local_file_path => local_file_path})
           end
         end
       end
-
       @hydra.run
-      failure = failure || @async_requests.any?{|request| request.handled_response == false }
 
-      failure ? last_checkpoint : (new_checkpoint || last_checkpoint)
+      pfailure = failure || @async_requests.any? do |request| 
+        @logger.warn "request did not handle response: #{request.inspect}" if request.response.nil? || request.response.code != 200
+        request.response.nil? || request.response.code != 200
+      end
+
+      if pfailure == false
+        @logger.info "Done fetching files for #{@repository.name}, processing callbacks..."
+        begin
+          
+          run_callbacks_delete(callbacks_on_delete)
+          
+          run_callbacks(callbacks_on_update)
+          
+          run_callbacks_no_change(callbacks_on_no_change)
+
+          new_checkpoint || last_checkpoint
+        rescue RuntimeError, SystemCallError, StandardError => e
+          msg = ["Callbacks failed to run; #{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+          @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}; #{msg}"
+          last_checkpoint
+        end
+      else
+        @logger.error "FAILED to update files for #{@repository.name}  last_checkpoint = #{last_checkpoint}"
+        last_checkpoint
+      end
     end
     
     def update_files!(last_checkpoint)
@@ -143,27 +167,26 @@ module Filbunke
         update_http_file!(file, local_file_path)
       elsif (file.url =~ /^hdfs:\/\//)
         success = update_hdfs_file!(file, local_file_path)
-        run_callbacks(file, local_file_path) if success
       else
         raise "Unsupported protocol for file: #{file.inspect}"
       end
     end
 
-    def run_callbacks(file, local_file_path)
+    def run_callbacks(files)
       @callbacks.each do |callback|
-        callback.on_update(file, local_file_path)
+        callback.on_update_batch(files)
       end
     end
 
-    def run_callbacks_no_change(file, local_file_path)
+    def run_callbacks_no_change(files)
       @callbacks.each do |callback|
-        callback.on_no_change(file, local_file_path)
+        callback.on_no_change_batch(files)
       end
     end
 
-    def run_callbacks_delete(file, local_file_path)
+    def run_callbacks_delete(files)
       @callbacks.each do |callback|
-        callback.on_delete(file, local_file_path)
+        callback.on_delete_batch(files)
       end
     end
 
@@ -177,27 +200,28 @@ module Filbunke
     end
 
     def get_updated_file_list(last_checkpoint)
-
       begin
         updates_http = Net::HTTP.new(@repository.host, @repository.port)
+        updates_http.read_timeout = 300 # default is 60 seconds
         updates_http.start do |http|
+          updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}"
           begin
-            updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}"
+            @logger.info "Fetching updated file list from #{updates_path}"
             request = Net::HTTP::Get.new(updates_path)
             response = http.request(request)
             if response.code.to_i == 200
               JSON.parse(response.body)
             else
-              @logger.log "Failed to download updates for #{@repository.name}, error code = #{response.code}"
+              @logger.error "Failed to download updates for #{@repository.name}, error code = #{response.code}"
               {}
             end
           rescue StandardError => e
-            @logger.log "Error getting file list: #{e.message}! Retrying later.."
+            @logger.error "Error getting file list from http://#{@repository.host}:#{@repository.port}#{updates_path}: #{e.message}! Retrying later.."
             {}
           end
         end
       rescue StandardError => e
-        @logger.log "Unable to create HTTP connection to #{@repository.host}:#{@repository.port} (#{e.message})!"
+        @logger.error "Unable to create HTTP connection to #{@repository.host}:#{@repository.port} (#{e.message})!"
         return {}
       end        
     end
@@ -205,32 +229,35 @@ module Filbunke
     def update_http_file!(file, local_file_path)
       begin
         async_request = if @repository.user
-          Typhoeus::Request.new(URI.encode(file.url), :username => @repository.user, :password => @repository.pass)
+          Typhoeus::Request.new(URI.encode(file.url), :follow_location => true, :username => @repository.user, :password => @repository.pass)
         else
-          Typhoeus::Request.new(URI.encode(file.url))
+          Typhoeus::Request.new(URI.encode(file.url), :follow_location => true)
         end
         async_request.on_complete do |response|
-          success = true
+          success = false
           begin
-            if response.code.to_i == 200
+            success = response.code.to_i == 200
+            if success
               write_file!(local_file_path, response.body)
             else
-              @logger.log "Failed to update file #{file.url}, error code = #{response.code}"
-              success = false
+              body_if_error = response.code >= 500 ? ", body = #{response.body}" : ""
+              @logger.warn "Failed to update file #{file.url}, got status code = #{response.code}#{body_if_error}"
             end
-            run_callbacks(file, local_file_path) if success
-          rescue StandardError => e
-            @logger.log "Failed to update file #{file.url}: #{e.message}"
-            success = false
+          rescue SystemCallError, StandardError => e
+            msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+            @logger.error "Failed to update file #{file.url}: #{msg}"
           end
+          # return the async_request.handled_response value here
           success
         end
         @hydra.queue async_request
         @async_requests << async_request
       rescue StandardError => e
-        @logger.log "Failed to update file #{file.url}: #{e.message}"
+        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "Failed to update file #{file.url}: #{msg}"
         return false
       end
+
       return true
     end
     
@@ -241,7 +268,7 @@ module Filbunke
         url = file.url
         url.gsub!(/hdfs:\/\/([^\/]*)(.*)/, "hdfs://\\2")
         hdfs_cmd = "#{@repository.hadoop_binary} dfs -copyToLocal #{url} #{local_file_path}.tmp"
-        @logger.log "Trying to update #{local_file_path} with '#{hdfs_cmd}'"
+        #@logger.debug "Trying to update #{local_file_path} with '#{hdfs_cmd}'"
         
         pid, stdin, stdout, stderr = Open4::popen4 hdfs_cmd
         ignored, status = Process::waitpid2 pid
@@ -251,38 +278,41 @@ module Filbunke
             ::FileUtils.mv "#{local_file_path}.tmp", local_file_path
             return true
           rescue StandardError => e
-            @logger.log "Failed to move hdfs file #{file.url}: #{e.message}"
+            msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+            @logger.error "Failed to move hdfs file #{file.url}: #{msg}"
             return false
           end  
         else
-          @logger.log "Failed to update hdfs file #{file.url}! Unable to execute #{hdfs_cmd}"
+          @logger.error "Failed to update hdfs file #{file.url}! Unable to execute #{hdfs_cmd}"
           return false
         end
-      rescue StandardError => e
-        @logger.log "Failed to update hdfs file #{file.url}: #{e.message}"
+      rescue SystemCallError, StandardError => e
+        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "Failed to update hdfs file #{file.url}: #{msg}"
         return false
       end
     end
 
     def write_file!(file_path, contents)
       ::FileUtils.mkdir_p(::File.dirname(file_path))
-      @logger.log("Updating: #{file_path}")
+      @logger.debug("Updating: #{file_path}")
       begin
         ::File.open("#{file_path}.tmp", 'w') do |file|
-          file.write(contents);
+          file.write(contents)
           file.close
         end
         ::FileUtils.mv "#{file_path}.tmp", file_path
         return true
       rescue StandardError => e
-        @logger.log "Failed to move file #{file.url}: #{e.message}"
+        msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+        @logger.error "Failed to move file #{file_path}: #{msg}"
         return false
       end  
     end
 
     def delete_file!(file_path)
       if ::File.exists?(file_path) then
-        @logger.log("Deleting: #{file_path}")
+        @logger.debug("Deleting: #{file_path}")
         ::File.delete(file_path)
       end
     end

data/lib/filbunke/daemon.rb

@@ -9,27 +9,26 @@ module Filbunke
     end
 
     def setup_clients!(local = false)
-      @logger = Logger.new(@config["log_file"], local)
+      @logger = FilbunkeLogger.new(@config["log_file"], local, @config["log_level"])
       @logger.log("Initializing filbunked")
       @config["repositories"].each do |repository_name, repository_config|
         @logger.log("Initializing repository: #{repository_name}")
         @clients << begin
-          repository = Repository.new(repository_config["filbunke_server_repository"],
-                                      repository_config["filbunke_server_host"],
-                                      repository_config["filbunke_server_port"],
-                                      repository_config["local_path"],
-                                      repository_config["file_umask"].to_i,
-                                      repository_config["directory_umask"].to_i,
-                                      repository_config["file_url_username"],
-                                      repository_config["file_url_password"],
-                                      repository_config["hadoop_binary"])
+          repository_config["run_every"] = repository_config.fetch("run_every", @config.fetch("run_every", 10))
+          repository = Repository.new(repository_config)
           callbacks = []
           repository_config["callbacks"].each do |callback_name, callback_config|
             require ::File.join(@config["callback_path"], callback_name.to_s)
             callback_class =  Module.const_get(callback_name.split("_").map(&:capitalize).join)
             callbacks << callback_class.new(@logger, callback_config)
           end
-          Client.new(repository, @logger, callbacks)
+          failed_request_log_file_name = repository_config["failed_request_log_file_name"]||nil
+          
+          Client.new(repository, @logger, callbacks, failed_request_log_file_name)
+        rescue Exception => e
+          msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+          @logger.error("Failed to initialize #{repository_name}; #{msg}")
+          raise e
         end
       end
     end
@@ -37,31 +36,44 @@ module Filbunke
     def run!
       version = ::File.read(::File.expand_path(::File.join(::File.dirname(__FILE__), "../../VERSION"))).chomp
       @logger.log("Starting filbunked version #{version}")
-      while true
-        begin
-          @clients.each do |client|
-            new_checkpoint = client.update_files!(checkpoint_for_repository(client.repository))
-            update_checkpoint_for_repository(client.repository, new_checkpoint)
+      client_pids = []
+      @parent_pid = Process.pid
+      @clients.each do |client|
+        client_pids << fork do 
+          begin
+            while process_is_alive(@parent_pid)
+              new_checkpoint = client.update_files!(checkpoint_for_repository(client.repository))
+              @logger.info "Update finished for #{client.repository.name} new checkpoint => #{new_checkpoint}" 
+              update_checkpoint_for_repository(client.repository, new_checkpoint)
+              sleep client.repository.run_every
+            end
+          rescue RuntimeError, SystemCallError, StandardError => e
+            msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n\t")
+            @logger.error("#{client.repository.name} Died..  #{msg}")
+            Process.kill("KILL", @parent_pid)
+            exit 1
           end
-        rescue StandardError => e
-          @logger.log("Died.. #{e.message}")
-          @logger.log(e.backtrace.join("\n"))
-          exit 1
         end
-        sleep @config["run_every"]
       end
+      client_pids.each { |pid| Process.wait(pid) }
     end
 
+    def process_is_alive(pid)
+      !!Process.kill(0, pid) rescue false
+    end
+    
     private
-
     def checkpoint_for_repository(repository)
       ::File.open(checkpoint_file_name_for_repository(repository), 'r') {|f| f.readline.to_i } rescue 0
     end
 
     def update_checkpoint_for_repository(repository, checkpoint)
       f = ::File.new(checkpoint_file_name_for_repository(repository), 'w', repository.file_umask)
-      f.write(checkpoint)
-      f.close
+      begin
+        f.write(checkpoint)
+      ensure
+        f.close
+      end
     end
 
     def checkpoint_file_name_for_repository(repository)
@@ -70,6 +82,13 @@ module Filbunke
     end
 
     def write_pid!(pid_file_path)
+      
+      existing_process = ::File.read(pid_file_path)
+      unless existing_process.nil?
+        @logger.info("killing existing process #{existing_process} from #{pid_file_path}")
+        Process.kill("KILL", existing_process.to_i) rescue nil
+      end
+
       ::File.open(pid_file_path, 'w') do |f|
         f.write(Process.pid.to_i)
         f.close

data/lib/filbunke/logger.rb

@@ -1,19 +1,58 @@
+require 'logger'
 module Filbunke
-  class Logger
+  class FilbunkeLogger
 
-    def initialize(log_file_name, local)
+    def initialize(log_file_name, local, level)
       @local = local
-      @log_file = ::File.open(log_file_name, "a") unless local
+      @log = if @local or log_file_name.nil?
+        Logger.new(STDOUT) 
+      else 
+        Logger.new(log_file_name)
+      end
+
+      @log.level = parse_level(level)
     end
 
+    def puts(msg)
+      info(msg)
+    end
+    
     def log(msg)
-      if @local then
-        puts "#{Time.now}: #{msg}"
-      else
-        @log_file.write("#{Time.now}: #{msg}\n")
-        @log_file.flush
-      end
+      @log.info msg
+    end
+
+    def info(msg)
+      @log.info msg
+    end
+
+    def error(msg)
+      @log.error msg
     end
 
+    def warn(msg)
+      @log.warn msg
+    end
+
+    def debug(msg)
+      @log.debug msg
+    end
+
+    def fatal(msg)
+      @log.error msg
+    end
+
+    def parse_level(constantOrString)
+      case constantOrString
+        when 'debug' then Logger::DEBUG
+        when 'info' then Logger::INFO
+        when 'warn' then Logger::WARN
+        when 'error' then Logger::ERROR
+        when 'fatal' then Logger::ERROR
+        when 'unknown' then Logger::UNKNOWN
+        when nil then Logger::INFO
+        else 
+          constantOrString
+      end
+    end
   end
 end

data/lib/filbunke/repository.rb

@@ -1,19 +1,31 @@
 module Filbunke
   class Repository
+    attr_accessor :name, 
+                  :host, 
+                  :port, 
+                  :local_path, 
+                  :file_umask, 
+                  :directory_umask, 
+                  :user, 
+                  :pass, 
+                  :hadoop_binary, 
+                  :run_every,
+                  :hydra_concurrency
+ 
+    def initialize(repository_config)
+      @name = repository_config["filbunke_server_repository"]
+      @host = repository_config["filbunke_server_host"]
+      @port = repository_config["filbunke_server_port"]
+      @local_path = repository_config["local_path"]
+      @file_umask = repository_config["file_umask"].to_i
+      @directory_umask = repository_config["directory_umask"].to_i
+      @user = repository_config["file_url_username"]
+      @pass = repository_config["file_url_password"]
+      @hadoop_binary = repository_config["hadoop_binary"]
+      @run_every = repository_config.fetch("run_every", 10).to_i
+      @hydra_concurrency = repository_config.fetch("hydra_concurrency", 100).to_i
+     end
+ 
+   end
 
-    attr_accessor :name, :host, :port, :local_path, :file_umask, :directory_umask, :user, :pass, :hadoop_binary
-
-    def initialize(name, host, port, local_path, file_umask, directory_umask, user = nil, pass = nil, hadoop_binary = nil)
-      @name = name
-      @host = host
-      @port = port
-      @local_path = local_path
-      @file_umask = file_umask
-      @directory_umask = directory_umask
-      @user = user
-      @pass = pass
-      @hadoop_binary = hadoop_binary
-    end
-
-  end
 end

data/lib/filbunke/thread_pool.rb

@@ -0,0 +1,104 @@
+# Inspired by https://github.com/meh/ruby-threadpool
+require 'thread'
+
+class ThreadPool
+
+  class Job < Struct.new(:args, :block); end
+
+  def initialize(min, max = nil)
+    
+    trap("INT") { shutdown }
+
+    @min = min
+    @max = max || min
+
+    @cv = ConditionVariable.new
+    @mutex = Mutex.new
+
+    @queue = []
+    @workers = []
+
+    @spawned = 0
+    @waiting = 0
+    @shutdown = false
+    @queue_locked = false
+
+    @mutex.synchronize do
+      min.times { spawn_thread }
+    end
+  end
+
+  def execute(*args, &block)
+    @mutex.synchronize do
+      raise "Thread pool is about to shutdown" if @shutdown || @queue_locked
+
+      @queue << Job.new(args, block)
+
+      spawn_thread if @waiting == 0 && @spawned < @max
+
+      @cv.signal
+    end
+  end
+  alias :<< :execute
+
+  def shutdown
+    @mutex.synchronize do
+      @shutdown = true
+      @cv.broadcast
+    end
+
+    @workers.first.join until @workers.empty?
+  end
+
+  def join
+    @mutex.synchronize do
+      @queue_locked = true
+      @cv.broadcast
+      sleep 0.01 until @queue.empty?
+    end
+    shutdown
+  end
+
+  protected
+
+  def spawn_thread
+    thread = Thread.new do
+      continue = true
+
+      while continue do
+        job = nil
+
+        @mutex.synchronize do
+          while @queue.empty? && continue
+            if @shutdown || @queue_locked
+              continue = false
+              break
+            end
+
+            @waiting += 1
+            @cv.wait @mutex
+            @waiting -= 1
+
+            if @shutdown || @queue_locked
+              continue = false
+              break
+            end
+          end
+
+          if continue
+            job = @queue.shift
+            job.block.call(*job.args) if job
+          end
+        end
+      end
+
+      @mutex.synchronize do
+        @spawned -= 1
+        @workers.delete thread
+      end
+    end
+
+    @workers << thread
+    thread
+  end
+end

metadata

@@ -1,28 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: filbunke
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.11.6
 platform: ruby
 authors:
+- Wouter de Bie
 - Bjorn Sperber
 - Karl Ravn
+- Magnus Spangdal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-12 00:00:00.000000000 Z
+date: 2015-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thoughtbot-shoulda
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -82,12 +84,16 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.19'
 description: Filbunke client and library
-email: info@deltaprojects.com
+email: technical@deltaprojects.com
 executables:
 - filbunked
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
+- .ruby-gemset
+- .ruby-version
+- Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -106,9 +112,10 @@ files:
 - lib/filbunke/file.rb
 - lib/filbunke/logger.rb
 - lib/filbunke/repository.rb
+- lib/filbunke/thread_pool.rb
 - test/helper.rb
 - test/test_filbunke.rb
-homepage: http://github.com/deltaprojects/filbunke
+homepage: https://rubygems.org/gems/filbunke
 licenses: []
 metadata: {}
 post_install_message: 
@@ -117,17 +124,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Filbunke client