filbunke 1.2.1 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/doc/examples/filbunke_config.yml +2 -1
- data/filbunke.gemspec +2 -2
- data/lib/filbunke/client.rb +47 -17
- data/lib/filbunke/daemon.rb +2 -1
- data/lib/filbunke/repository.rb +3 -2
- metadata +4 -4
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.3.0
|
data/filbunke.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{filbunke}
|
8
|
-
s.version = "1.
|
8
|
+
s.version = "1.3.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Wouter de Bie"]
|
12
|
-
s.date = %q{2010-11-
|
12
|
+
s.date = %q{2010-11-19}
|
13
13
|
s.default_executable = %q{filbunked}
|
14
14
|
s.description = %q{Filbunke client and library}
|
15
15
|
s.email = %q{wouter@deltaprojects.se}
|
data/lib/filbunke/client.rb
CHANGED
@@ -32,25 +32,25 @@ module Filbunke
|
|
32
32
|
|
33
33
|
local_file_path = ::File.join(repository.local_path, file.path)
|
34
34
|
if file_needs_update?(file, local_file_path)
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
end
|
40
|
-
yield file
|
41
|
-
else
|
42
|
-
@logger.log "Unable to get file #{file.url} ==> #{file.path}!"
|
43
|
-
failure = true
|
35
|
+
|
36
|
+
if update_file!(file, local_file_path) then
|
37
|
+
@callbacks.each do |callback|
|
38
|
+
callback.on_update(file)
|
44
39
|
end
|
40
|
+
yield file
|
45
41
|
else
|
46
|
-
|
42
|
+
@logger.log "Unable to get file #{file.url} ==> #{file.path}!"
|
43
|
+
failure = true
|
47
44
|
end
|
45
|
+
|
48
46
|
else
|
49
47
|
@logger.log "File exists with correct hash: #{local_file_path}"
|
50
48
|
end
|
51
49
|
end
|
52
50
|
failure ? last_checkpoint : (new_checkpoint || last_checkpoint)
|
53
51
|
end
|
52
|
+
|
53
|
+
|
54
54
|
|
55
55
|
def update_files!(last_checkpoint)
|
56
56
|
with_updated_files(last_checkpoint) {}
|
@@ -108,6 +108,22 @@ module Filbunke
|
|
108
108
|
|
109
109
|
private
|
110
110
|
|
111
|
+
def update_file!(file, local_file_path)
|
112
|
+
|
113
|
+
if file.state == "DELETED" then
|
114
|
+
delete_file!(local_file_path)
|
115
|
+
return true
|
116
|
+
end
|
117
|
+
|
118
|
+
if file.url =~ /^http:\/\//
|
119
|
+
update_http_file!(file, local_file_path)
|
120
|
+
elsif file.url =~ /^hdfs:\/\//
|
121
|
+
update_hdfs_file!(file, local_file_path)
|
122
|
+
else
|
123
|
+
raise "Unsupported protocol for file: #{file.inspect}"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
111
127
|
def file_needs_update?(file, local_file_path)
|
112
128
|
return true if file.hash.nil? || file.hash == ""
|
113
129
|
|
@@ -144,13 +160,7 @@ module Filbunke
|
|
144
160
|
end
|
145
161
|
|
146
162
|
def update_http_file!(file, local_file_path)
|
147
|
-
begin
|
148
|
-
|
149
|
-
if file.state == "DELETED" then
|
150
|
-
delete_file!(local_file_path)
|
151
|
-
return true
|
152
|
-
end
|
153
|
-
|
163
|
+
begin
|
154
164
|
uri = URI.parse(file.url)
|
155
165
|
file_http=Net::HTTP.new(uri.host, uri.port)
|
156
166
|
file_http.start do |http|
|
@@ -171,6 +181,26 @@ module Filbunke
|
|
171
181
|
return false
|
172
182
|
end
|
173
183
|
end
|
184
|
+
|
185
|
+
def update_hdfs_file!(file, local_file_path)
|
186
|
+
begin
|
187
|
+
::FileUtils.mkdir_p(::File.dirname(local_file_path))
|
188
|
+
|
189
|
+
hdfs_cmd = "#{@repository.hadoop_binary} fs -copyToLocal #{file.url} #{local_file_path}.tmp &> /dev/null && mv #{local_file_path}.tmp #{local_file_path}"
|
190
|
+
|
191
|
+
system hdfs_cmd
|
192
|
+
|
193
|
+
if $?.exitstatus == 0 then
|
194
|
+
return true
|
195
|
+
else
|
196
|
+
@logger.log "Failed to update file #{file.url}!"
|
197
|
+
return false
|
198
|
+
end
|
199
|
+
rescue StandardError => e
|
200
|
+
@logger.log "Failed to update file #{file.url}: #{e.message}"
|
201
|
+
return false
|
202
|
+
end
|
203
|
+
end
|
174
204
|
|
175
205
|
def write_file!(file_path, contents)
|
176
206
|
if ::File.exists?(file_path)
|
data/lib/filbunke/daemon.rb
CHANGED
@@ -21,7 +21,8 @@ module Filbunke
|
|
21
21
|
repository_config["file_umask"].to_i,
|
22
22
|
repository_config["directory_umask"].to_i,
|
23
23
|
repository_config["file_url_username"],
|
24
|
-
repository_config["file_url_password"]
|
24
|
+
repository_config["file_url_password"],
|
25
|
+
repository_config["hadoop_binary"])
|
25
26
|
callbacks = []
|
26
27
|
repository_config["callbacks"].each do |callback_name, callback_config|
|
27
28
|
require ::File.join(@config["callback_path"], callback_name.to_s)
|
data/lib/filbunke/repository.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
module Filbunke
|
2
2
|
class Repository
|
3
3
|
|
4
|
-
attr_accessor :name, :host, :port, :local_path, :file_umask, :directory_umask, :user, :pass
|
4
|
+
attr_accessor :name, :host, :port, :local_path, :file_umask, :directory_umask, :user, :pass, :hadoop_binary
|
5
5
|
|
6
|
-
def initialize(name, host, port, local_path, file_umask, directory_umask, user = nil, pass = nil)
|
6
|
+
def initialize(name, host, port, local_path, file_umask, directory_umask, user = nil, pass = nil, hadoop_binary = nil)
|
7
7
|
@name = name
|
8
8
|
@host = host
|
9
9
|
@port = port
|
@@ -12,6 +12,7 @@ module Filbunke
|
|
12
12
|
@directory_umask = directory_umask
|
13
13
|
@user = user
|
14
14
|
@pass = pass
|
15
|
+
@hadoop_binary = hadoop_binary
|
15
16
|
end
|
16
17
|
|
17
18
|
end
|
metadata
CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
|
|
4
4
|
prerelease: false
|
5
5
|
segments:
|
6
6
|
- 1
|
7
|
-
-
|
8
|
-
-
|
9
|
-
version: 1.
|
7
|
+
- 3
|
8
|
+
- 0
|
9
|
+
version: 1.3.0
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Wouter de Bie
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-11-
|
17
|
+
date: 2010-11-19 00:00:00 +01:00
|
18
18
|
default_executable: filbunked
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|