filbunke 1.11.9 → 1.12.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NTE5NTY2YzFkNmI4MmEyNTNkNGQxMmFkNGMyZjk5MTk1YjFhN2JlOA==
+    MzE3ODU4ZGQzMmIyYzA1MzI0MmVkZTY4YWNkNTE2NDk3MGQ4Y2RmYw==
   data.tar.gz: !binary |-
-    NDQzZTY0ZWE4MjM0ZDhjM2ZlNjgwYjk2OTc3NDcxNDdlNTZlYTZmMg==
+    YTQ1ZGNlYTVmYTNhY2RmY2UyNmFiZWVkYzIxZThmYTQ4NTlhZTVmMw==
 SHA512:
   metadata.gz: !binary |-
-    NmUwNDEyZWUzYjdkZmQ4OWU4NGRkZWEzZTcyY2ZiZjZiOTA0ODJkZjZjZWZm
-    MjBmN2Q3ZDA4YjNiYjZhMTg0YWZlZTAwNjQ5OTJmZTA1NzhkOTA2NmJlYjgx
-    MDkwYzZjMjZmMDZmZWYwOGMxNDc2YThmNTMxZjE2OGUyMTdiOTQ=
+    Mzc2NmU2YzRhNDJhNzNkNDFhZTI2MDg2NzU3ZWU3MDkyZmM5ODU5ZTFlMDBi
+    Y2FhM2QxZGQ2ZjkyNmRmZGY0ODNlMzFmMmI1MzZlNGYzNTRjN2E0ODllNDhk
+    MWEzOTQ2OTFlZGQ1MDk3NTcwNDk5YzUzYzUyODliZjNiMmQxMGQ=
   data.tar.gz: !binary |-
-    NTIyYzc4Yzg5YmE1MWFiMjliZTdkNDVkYjViMmM4NjQzNDgxNjdlMWYxNjJk
-    NjYzMDBhMTJmNTA3ZWE3NmQzYTBlOWQ0ZjQ5MjVjM2RiNjBlN2Y4YzUyNzIx
-    MTUyMzk5NzIxYTcwYzYxMDQ5YWVjZGRiMzZlODJhNTcyNDA0MGI=
+    NDRjNzVjN2FmZDgzODYzMjNiYzM4NGQ3YzkzNTJjNGVhMmU4M2NjN2Y3YTky
+    YzU2MmIzNDE4ZWNmZGJkM2NiZmQwNDk1MDQ5ZTkwNDE3M2JjNWY4ZTFhYmIz
+    N2IwMTE1MjQwM2M2M2RiOTM2YmJhOTQzNjcwNmUxYTkzNWMwODI=

data/VERSION

@@ -1 +1 @@
-1.11.9
+1.12.0

data/filbunke.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: filbunke 1.11.9 ruby lib
+# stub: filbunke 1.12.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "filbunke"
-  s.version = "1.11.9"
+  s.version = "1.12.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
-  s.date = "2015-11-05"
+  s.date = "2016-03-04"
   s.description = "Filbunke client and library"
   s.email = "technical@deltaprojects.com"
   s.executables = ["filbunked"]

data/lib/filbunke/client.rb

@@ -10,7 +10,6 @@ module Filbunke
     URL_KEY = 'url'
     FROM_CHECKPOINT_KEY = 'from_checkpoint'
     HASH_KEY = 'hash'
-    URI_UNSAFE_CHARACTERS = '/[^.:\/\w-]/'
 
 
     def initialize(repository, logger, callbacks = [], failed_request_log_file_name = nil)
@@ -206,6 +205,7 @@ module Filbunke
         updates_http.read_timeout = 300 # default is 60 seconds
         updates_http.start do |http|
           updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}"
+          updates_path = "#{updates_path}&batch_size=#{@repository.batch_size}" if @repository.batch_size
           begin
             @logger.info "Fetching updated file list from #{updates_path}"
             request = Net::HTTP::Get.new(updates_path)
@@ -224,15 +224,15 @@ module Filbunke
       rescue StandardError => e
         @logger.error "Unable to create HTTP connection to #{@repository.host}:#{@repository.port} (#{e.message})!"
         return {}
-      end
+      end        
     end
 
     def update_http_file!(file, local_file_path)
       begin
         async_request = if @repository.user
-          Typhoeus::Request.new(URI.encode(file.url, URI_UNSAFE_CHARACTERS), :follow_location => true, :username => @repository.user, :password => @repository.pass)
+          Typhoeus::Request.new(URI.encode(file.url), :follow_location => true, :username => @repository.user, :password => @repository.pass)
         else
-          Typhoeus::Request.new(URI.encode(file.url, URI_UNSAFE_CHARACTERS), :follow_location => true)
+          Typhoeus::Request.new(URI.encode(file.url), :follow_location => true)
         end
         async_request.on_complete do |response|
           success = false

data/lib/filbunke/daemon.rb

@@ -15,6 +15,7 @@ module Filbunke
         @logger.log("Initializing repository: #{repository_name}")
         @clients << begin
           repository_config["run_every"] = repository_config.fetch("run_every", @config.fetch("run_every", 10))
+          repository_config["batch_size"] = repository_config.fetch("batch_size", @config.fetch("batch_size", 0))
           repository = Repository.new(repository_config)
           callbacks = []
           repository_config["callbacks"].each do |callback_name, callback_config|

data/lib/filbunke/repository.rb

@@ -10,7 +10,8 @@ module Filbunke
                   :pass, 
                   :hadoop_binary, 
                   :run_every,
-                  :hydra_concurrency
+                  :hydra_concurrency,
+                  :batch_size
  
     def initialize(repository_config)
       @name = repository_config["filbunke_server_repository"]
@@ -24,8 +25,8 @@ module Filbunke
       @hadoop_binary = repository_config["hadoop_binary"]
       @run_every = repository_config.fetch("run_every", 10).to_i
       @hydra_concurrency = repository_config.fetch("hydra_concurrency", 100).to_i
-     end
- 
+      # batch_size == 0 means use default configured in filbunke-server
+      @batch_size = repository_config.fetch("batch_size", 0).to_i
    end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filbunke
 version: !ruby/object:Gem::Version
-  version: 1.11.9
+  version: 1.12.0
 platform: ruby
 authors:
 - Wouter de Bie
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-05 00:00:00.000000000 Z
+date: 2016-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thoughtbot-shoulda