filbunke 1.11.9 → 1.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/VERSION +1 -1
- data/filbunke.gemspec +3 -3
- data/lib/filbunke/client.rb +4 -4
- data/lib/filbunke/daemon.rb +1 -0
- data/lib/filbunke/repository.rb +4 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MzE3ODU4ZGQzMmIyYzA1MzI0MmVkZTY4YWNkNTE2NDk3MGQ4Y2RmYw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YTQ1ZGNlYTVmYTNhY2RmY2UyNmFiZWVkYzIxZThmYTQ4NTlhZTVmMw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
Mzc2NmU2YzRhNDJhNzNkNDFhZTI2MDg2NzU3ZWU3MDkyZmM5ODU5ZTFlMDBi
|
10
|
+
Y2FhM2QxZGQ2ZjkyNmRmZGY0ODNlMzFmMmI1MzZlNGYzNTRjN2E0ODllNDhk
|
11
|
+
MWEzOTQ2OTFlZGQ1MDk3NTcwNDk5YzUzYzUyODliZjNiMmQxMGQ=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NDRjNzVjN2FmZDgzODYzMjNiYzM4NGQ3YzkzNTJjNGVhMmU4M2NjN2Y3YTky
|
14
|
+
YzU2MmIzNDE4ZWNmZGJkM2NiZmQwNDk1MDQ5ZTkwNDE3M2JjNWY4ZTFhYmIz
|
15
|
+
N2IwMTE1MjQwM2M2M2RiOTM2YmJhOTQzNjcwNmUxYTkzNWMwODI=
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.12.0
|
data/filbunke.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: filbunke 1.
|
5
|
+
# stub: filbunke 1.12.0 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "filbunke"
|
9
|
-
s.version = "1.
|
9
|
+
s.version = "1.12.0"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Wouter de Bie", "Bjorn Sperber", "Karl Ravn", "Magnus Spangdal"]
|
14
|
-
s.date = "
|
14
|
+
s.date = "2016-03-04"
|
15
15
|
s.description = "Filbunke client and library"
|
16
16
|
s.email = "technical@deltaprojects.com"
|
17
17
|
s.executables = ["filbunked"]
|
data/lib/filbunke/client.rb
CHANGED
@@ -10,7 +10,6 @@ module Filbunke
|
|
10
10
|
URL_KEY = 'url'
|
11
11
|
FROM_CHECKPOINT_KEY = 'from_checkpoint'
|
12
12
|
HASH_KEY = 'hash'
|
13
|
-
URI_UNSAFE_CHARACTERS = '/[^.:\/\w-]/'
|
14
13
|
|
15
14
|
|
16
15
|
def initialize(repository, logger, callbacks = [], failed_request_log_file_name = nil)
|
@@ -206,6 +205,7 @@ module Filbunke
|
|
206
205
|
updates_http.read_timeout = 300 # default is 60 seconds
|
207
206
|
updates_http.start do |http|
|
208
207
|
updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}"
|
208
|
+
updates_path = "#{updates_path}&batch_size=#{@repository.batch_size}" if @repository.batch_size
|
209
209
|
begin
|
210
210
|
@logger.info "Fetching updated file list from #{updates_path}"
|
211
211
|
request = Net::HTTP::Get.new(updates_path)
|
@@ -224,15 +224,15 @@ module Filbunke
|
|
224
224
|
rescue StandardError => e
|
225
225
|
@logger.error "Unable to create HTTP connection to #{@repository.host}:#{@repository.port} (#{e.message})!"
|
226
226
|
return {}
|
227
|
-
end
|
227
|
+
end
|
228
228
|
end
|
229
229
|
|
230
230
|
def update_http_file!(file, local_file_path)
|
231
231
|
begin
|
232
232
|
async_request = if @repository.user
|
233
|
-
Typhoeus::Request.new(URI.encode(file.url
|
233
|
+
Typhoeus::Request.new(URI.encode(file.url), :follow_location => true, :username => @repository.user, :password => @repository.pass)
|
234
234
|
else
|
235
|
-
Typhoeus::Request.new(URI.encode(file.url
|
235
|
+
Typhoeus::Request.new(URI.encode(file.url), :follow_location => true)
|
236
236
|
end
|
237
237
|
async_request.on_complete do |response|
|
238
238
|
success = false
|
data/lib/filbunke/daemon.rb
CHANGED
@@ -15,6 +15,7 @@ module Filbunke
|
|
15
15
|
@logger.log("Initializing repository: #{repository_name}")
|
16
16
|
@clients << begin
|
17
17
|
repository_config["run_every"] = repository_config.fetch("run_every", @config.fetch("run_every", 10))
|
18
|
+
repository_config["batch_size"] = repository_config.fetch("batch_size", @config.fetch("batch_size", 0))
|
18
19
|
repository = Repository.new(repository_config)
|
19
20
|
callbacks = []
|
20
21
|
repository_config["callbacks"].each do |callback_name, callback_config|
|
data/lib/filbunke/repository.rb
CHANGED
@@ -10,7 +10,8 @@ module Filbunke
|
|
10
10
|
:pass,
|
11
11
|
:hadoop_binary,
|
12
12
|
:run_every,
|
13
|
-
:hydra_concurrency
|
13
|
+
:hydra_concurrency,
|
14
|
+
:batch_size
|
14
15
|
|
15
16
|
def initialize(repository_config)
|
16
17
|
@name = repository_config["filbunke_server_repository"]
|
@@ -24,8 +25,8 @@ module Filbunke
|
|
24
25
|
@hadoop_binary = repository_config["hadoop_binary"]
|
25
26
|
@run_every = repository_config.fetch("run_every", 10).to_i
|
26
27
|
@hydra_concurrency = repository_config.fetch("hydra_concurrency", 100).to_i
|
27
|
-
|
28
|
-
|
28
|
+
# batch_size == 0 means use default configured in filbunke-server
|
29
|
+
@batch_size = repository_config.fetch("batch_size", 0).to_i
|
29
30
|
end
|
30
31
|
|
31
32
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filbunke
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.12.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Wouter de Bie
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date:
|
14
|
+
date: 2016-03-04 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: thoughtbot-shoulda
|