mogilefs-client 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +3 -0
- data/.gitignore +1 -0
- data/GIT-VERSION-GEN +1 -1
- data/GNUmakefile +2 -10
- data/History +3 -0
- data/Rakefile +8 -0
- data/bin/mog +18 -38
- data/examples/stale_fid_checker.rb +108 -0
- data/lib/mogilefs.rb +3 -1
- data/lib/mogilefs/admin.rb +2 -1
- data/lib/mogilefs/backend.rb +16 -44
- data/lib/mogilefs/chunker.rb +11 -5
- data/lib/mogilefs/client.rb +6 -1
- data/lib/mogilefs/http_file.rb +51 -88
- data/lib/mogilefs/mogilefs.rb +91 -39
- data/lib/mogilefs/new_file.rb +78 -0
- data/lib/mogilefs/new_file/common.rb +89 -0
- data/lib/mogilefs/new_file/content_range.rb +105 -0
- data/lib/mogilefs/new_file/stream.rb +91 -0
- data/lib/mogilefs/new_file/tempfile.rb +24 -0
- data/lib/mogilefs/new_file/writer.rb +57 -0
- data/lib/mogilefs/pool.rb +18 -10
- data/lib/mogilefs/socket/kgio.rb +4 -3
- data/lib/mogilefs/socket/pure_ruby.rb +3 -3
- data/lib/mogilefs/socket_common.rb +7 -6
- data/test/fresh.rb +5 -2
- data/test/test_backend.rb +16 -8
- data/test/test_mogilefs_integration.rb +121 -0
- data/test/test_mogstored_rack.rb +112 -6
- data/test/test_pool.rb +26 -0
- metadata +13 -10
- data/Manifest.txt +0 -60
- data/examples/mogstored_rack.rb +0 -188
- data/test/test_unit_mogstored_rack.rb +0 -72
data/.document
CHANGED
data/.gitignore
CHANGED
data/GIT-VERSION-GEN
CHANGED
data/GNUmakefile
CHANGED
@@ -34,15 +34,7 @@ $(T):
|
|
34
34
|
$(run_test)
|
35
35
|
|
36
36
|
RUBY_VERSION_FILE = lib/mogilefs/version.rb
|
37
|
-
|
38
|
-
manifest: Manifest.txt
|
39
|
-
Manifest.txt:
|
40
|
-
git ls-files > $@+
|
41
|
-
echo $(RUBY_VERSION_FILE) >> $@+
|
42
|
-
cmp $@+ $@ || mv $@+ $@
|
43
|
-
$(RM) -f $@+
|
44
|
-
|
45
|
-
package: manifest
|
37
|
+
package:
|
46
38
|
git diff --exit-code HEAD^0
|
47
39
|
$(RM) -r pkg/
|
48
40
|
rake fix_perms
|
@@ -55,7 +47,7 @@ flay: $(libs)
|
|
55
47
|
flay $(flay_flags) $^
|
56
48
|
flog: $(libs)
|
57
49
|
flog $(flog_flags) $^
|
58
|
-
.PHONY: $(T)
|
50
|
+
.PHONY: $(T)
|
59
51
|
|
60
52
|
check-warnings:
|
61
53
|
@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \
|
data/History
CHANGED
data/Rakefile
CHANGED
@@ -2,10 +2,18 @@ require 'rubygems'
|
|
2
2
|
require 'hoe'
|
3
3
|
load "./GIT-VERSION-GEN"
|
4
4
|
|
5
|
+
include Rake::DSL if defined?(Rake::DSL)
|
5
6
|
$:.unshift 'lib'
|
6
7
|
require 'mogilefs'
|
7
8
|
Hoe.plugin :seattlerb
|
8
9
|
|
10
|
+
manifest = "Manifest.txt"
|
11
|
+
if ! File.exist?(manifest) ||
|
12
|
+
File.stat(manifest).mtime < File.stat(RVF).mtime
|
13
|
+
system("git ls-files > #{manifest}")
|
14
|
+
File.open(manifest, "a") { |fp| fp.puts("lib/mogilefs/version.rb") }
|
15
|
+
end
|
16
|
+
|
9
17
|
Hoe.spec 'mogilefs-client' do
|
10
18
|
self.rubyforge_name = 'seattlerb'
|
11
19
|
developer 'Eric Wong', 'normalperson@yhbt.net'
|
data/bin/mog
CHANGED
@@ -14,8 +14,7 @@ if md5_trailer_nodes = ENV["MD5_TRAILER_NODES"]
|
|
14
14
|
end
|
15
15
|
|
16
16
|
# this is to be compatible with config files used by the Perl tools
|
17
|
-
def parse_config_file!(path,
|
18
|
-
dest = {}
|
17
|
+
def parse_config_file!(path, dest = {})
|
19
18
|
File.open(path).each_line do |line|
|
20
19
|
case line
|
21
20
|
when /^(domain|class)\s*=\s*(\S+)/
|
@@ -43,6 +42,7 @@ config_file = nil
|
|
43
42
|
ls_l = false
|
44
43
|
ls_h = false
|
45
44
|
chunk = false
|
45
|
+
range = false
|
46
46
|
test = {}
|
47
47
|
cat = { :raw => false }
|
48
48
|
|
@@ -70,6 +70,7 @@ ARGV.options do |x|
|
|
70
70
|
x.on('-h', '--human-readable',
|
71
71
|
"print sizes in human-readable format (`ls' command)") { ls_h = true }
|
72
72
|
x.on('--chunk', "chunk uploads (`tee' command)") { chunk = true }
|
73
|
+
x.on('--range', "stream partial uploads (`tee' command)") { range = true }
|
73
74
|
x.separator ''
|
74
75
|
x.on('--help', 'Show this help message.') { puts x; exit }
|
75
76
|
x.on('--version', 'Show --version') { puts "#$0 #{MogileFS::VERSION}"; exit }
|
@@ -77,7 +78,7 @@ ARGV.options do |x|
|
|
77
78
|
end
|
78
79
|
|
79
80
|
# parse the config file specified at the command-line
|
80
|
-
file_cfg = config_file ? parse_config_file!(config_file
|
81
|
+
file_cfg = config_file ? parse_config_file!(config_file) : {}
|
81
82
|
|
82
83
|
# read environment variables, too. This Ruby API favors the term
|
83
84
|
# "hosts", however upstream MogileFS teminology favors "trackers" instead.
|
@@ -188,6 +189,7 @@ begin
|
|
188
189
|
puts "Key: #{key}"
|
189
190
|
puts "Size: #{info['length']}"
|
190
191
|
puts "Class: #{info['class']}"
|
192
|
+
checksum = info['checksum'] and puts "Checksum: #{checksum}"
|
191
193
|
o = { :pathcount => info["devcount"] }
|
192
194
|
mg.get_paths(key, o).each_with_index do |path,i|
|
193
195
|
puts "URL-#{i}: #{path}"
|
@@ -200,49 +202,27 @@ begin
|
|
200
202
|
end
|
201
203
|
exit(ok)
|
202
204
|
when 'tee'
|
203
|
-
|
205
|
+
abort "--range and --chunk are incompatible" if range && chunk
|
204
206
|
dkey = ARGV.shift or raise ArgumentError, '<key>'
|
205
207
|
ARGV.shift and raise ArgumentError, '<key>'
|
206
208
|
cfg[:noclobber] && mg.exist?(dkey) and
|
207
209
|
abort "`#{dkey}' already exists and -n/--no-clobber was specified"
|
208
210
|
skip_tee = File.stat('/dev/null') == $stdout.stat
|
211
|
+
largefile = :tempfile
|
212
|
+
largefile = :content_range if range
|
213
|
+
largefile = :stream if chunk
|
209
214
|
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
else
|
214
|
-
tee_obj = lambda do |*args|
|
215
|
-
buf = $stdin.readpartial(*args)
|
216
|
-
$stdout.write(buf)
|
217
|
-
buf
|
218
|
-
end
|
219
|
-
class << tee_obj
|
220
|
-
alias readpartial call
|
221
|
-
end
|
222
|
-
end
|
223
|
-
mg.store_file(dkey, cfg[:class], tee_obj)
|
224
|
-
else # buffer input, first
|
225
|
-
tmp = Tempfile.new('mog-tee')
|
226
|
-
tmp.sync = true
|
227
|
-
|
228
|
-
# if stdout is pointing to /dev/null, don't bother installing the filter.
|
229
|
-
tee_obj = tmp
|
230
|
-
unless skip_tee
|
231
|
-
tee_obj = lambda do |buf|
|
232
|
-
$stdout.write(buf)
|
233
|
-
tmp.write(buf)
|
234
|
-
end
|
235
|
-
class << tee_obj
|
236
|
-
alias write call
|
237
|
-
end
|
238
|
-
end
|
215
|
+
io = mg.new_file(dkey, :class => cfg[:class], :largefile => largefile)
|
216
|
+
begin
|
217
|
+
buf = $stdin.readpartial(16384)
|
239
218
|
begin
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
219
|
+
io.write(buf)
|
220
|
+
$stdout.write(buf) unless skip_tee
|
221
|
+
$stdin.readpartial(16384, buf)
|
222
|
+
end while true
|
223
|
+
rescue EOFError
|
245
224
|
end
|
225
|
+
io.close
|
246
226
|
when 'test'
|
247
227
|
truth, ok = true, nil
|
248
228
|
raise ArgumentError, "-e must be specified" unless (test.size == 1)
|
@@ -0,0 +1,108 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# This requires:
|
3
|
+
# * net-http-persistent RubyGem
|
4
|
+
# * Ruby 1.9.2+
|
5
|
+
# * upstream MogileFS::Server 2.45 or later
|
6
|
+
$stdout.sync = $stderr.sync = true
|
7
|
+
usage = <<EOF
|
8
|
+
Usage: #$0 -t TRACKERS"
|
9
|
+
|
10
|
+
The output of this script can be piped to awk + curl to DELETE the files:
|
11
|
+
#$0 -t TRACKERS | awk '{system("curl -XDELETE "$3)}'
|
12
|
+
EOF
|
13
|
+
|
14
|
+
require 'uri'
|
15
|
+
require 'optparse'
|
16
|
+
require 'mogilefs'
|
17
|
+
require 'net/http/persistent'
|
18
|
+
Thread.abort_on_exception = true
|
19
|
+
MogileFS::VERSION <= "3.0.0" and
|
20
|
+
abort "Upgrade mogilefs-client (to a version that distributes this script)" \
|
21
|
+
"MogileFS::Admin#each_fid is probably broken in this version"
|
22
|
+
|
23
|
+
trackers = []
|
24
|
+
ARGV.options do |x|
|
25
|
+
x.banner = usage.strip
|
26
|
+
x.separator ''
|
27
|
+
x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]',
|
28
|
+
Array, 'hostnames/IP addresses of trackers') do |args|
|
29
|
+
trackers = args
|
30
|
+
end
|
31
|
+
|
32
|
+
x.on('-h', '--help', 'Show this help message.') { puts x; exit }
|
33
|
+
x.parse!
|
34
|
+
end
|
35
|
+
|
36
|
+
adm = MogileFS::Admin.new(:hosts => trackers)
|
37
|
+
NHP = Net::HTTP::Persistent.new(File.basename($0))
|
38
|
+
client = MogileFS::MogileFS.new(:hosts => trackers, :domain => "none")
|
39
|
+
|
40
|
+
def start_perdev_thread(pfx)
|
41
|
+
todo = Queue.new
|
42
|
+
done = Queue.new
|
43
|
+
Thread.new do
|
44
|
+
while fid_path = todo.shift
|
45
|
+
path = "#{pfx}#{fid_path}"
|
46
|
+
uri = URI.parse(path)
|
47
|
+
begin
|
48
|
+
resp = NHP.request(uri, Net::HTTP::Head.new(uri.path))
|
49
|
+
done << [ path, resp ]
|
50
|
+
rescue => err
|
51
|
+
done << [ path, err ]
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
[ todo, done ]
|
56
|
+
end
|
57
|
+
|
58
|
+
def setup_devices(dev_map, adm)
|
59
|
+
hosts = {}
|
60
|
+
adm.get_hosts.each do |host|
|
61
|
+
hosts[host["hostid"]] = "http://#{host['hostip']}:#{host['http_port']}/"
|
62
|
+
end
|
63
|
+
|
64
|
+
adm.get_devices.each do |device|
|
65
|
+
pfx = hosts[device["hostid"]] + "dev#{device['devid']}"
|
66
|
+
todo, done = start_perdev_thread(pfx)
|
67
|
+
dev_map[todo] = done
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def check(bad, curfid, rv)
|
72
|
+
path, resp = rv
|
73
|
+
case resp
|
74
|
+
when Net::HTTPNotFound # good
|
75
|
+
when Net::HTTPOK
|
76
|
+
bad << "#{curfid} #{resp.content_length} #{path}\n"
|
77
|
+
else
|
78
|
+
warn "E: #{resp.inspect} (#{resp.class}) #{path}"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
dev_map = {}
|
83
|
+
setup_devices(dev_map, adm)
|
84
|
+
next_fid = 0
|
85
|
+
adm.each_fid do |fid|
|
86
|
+
fidid = fid["fid"]
|
87
|
+
|
88
|
+
if fidid != next_fid
|
89
|
+
(next_fid..(fidid - 1)).each do |curfid|
|
90
|
+
nfid = sprintf("%010u", curfid)
|
91
|
+
/\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
|
92
|
+
fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
|
93
|
+
bad = []
|
94
|
+
dev_map.each_key { |todo| todo << fid_path }
|
95
|
+
dev_map.each_value { |done| check(bad, curfid, done.shift) }
|
96
|
+
next if bad.empty?
|
97
|
+
|
98
|
+
begin
|
99
|
+
info = client.file_debug(curfid)
|
100
|
+
abort "BUG: #{info.inspect} found!" if info["fid_dkey"]
|
101
|
+
rescue MogileFS::Backend::UnknownFidError
|
102
|
+
end
|
103
|
+
|
104
|
+
puts bad.join
|
105
|
+
end
|
106
|
+
end
|
107
|
+
next_fid = fidid + 1
|
108
|
+
end
|
data/lib/mogilefs.rb
CHANGED
data/lib/mogilefs/admin.rb
CHANGED
@@ -12,7 +12,7 @@ class MogileFS::Admin < MogileFS::Client
|
|
12
12
|
low = -1
|
13
13
|
rv = 0
|
14
14
|
begin
|
15
|
-
fids = list_fids(low
|
15
|
+
fids = list_fids(low)
|
16
16
|
fids.each { |fid| yield fid }
|
17
17
|
rv += fids.size
|
18
18
|
end while last = fids[-1] and low = last["fid"]
|
@@ -273,6 +273,7 @@ class MogileFS::Admin < MogileFS::Client
|
|
273
273
|
rv
|
274
274
|
end
|
275
275
|
|
276
|
+
# Clears the tracker caches. Not implemented in all versions of MogileFS
|
276
277
|
def clear_cache
|
277
278
|
@backend.clear_cache
|
278
279
|
end
|
data/lib/mogilefs/backend.rb
CHANGED
@@ -42,6 +42,14 @@ class MogileFS::Backend
|
|
42
42
|
BACKEND_ERRORS[err_snake] = const_get(err_camel)
|
43
43
|
end
|
44
44
|
|
45
|
+
def self.const_missing(name) # :nodoc:
|
46
|
+
if /Error\z/ =~ name.to_s
|
47
|
+
const_set(name, Class.new(MogileFS::Error))
|
48
|
+
else
|
49
|
+
super name
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
45
53
|
##
|
46
54
|
# The last error
|
47
55
|
|
@@ -62,6 +70,7 @@ class MogileFS::Backend
|
|
62
70
|
|
63
71
|
def initialize(args)
|
64
72
|
@hosts = args[:hosts]
|
73
|
+
@fail_timeout = args[:fail_timeout] || 5
|
65
74
|
raise ArgumentError, "must specify at least one host" unless @hosts
|
66
75
|
raise ArgumentError, "must specify at least one host" if @hosts.empty?
|
67
76
|
unless @hosts == @hosts.select { |h| h =~ /:\d+$/ } then
|
@@ -90,6 +99,7 @@ class MogileFS::Backend
|
|
90
99
|
add_command :create_open
|
91
100
|
add_command :create_close
|
92
101
|
add_idempotent_command :get_paths
|
102
|
+
add_idempotent_command :noop
|
93
103
|
add_command :delete
|
94
104
|
add_idempotent_command :sleep
|
95
105
|
add_command :rename
|
@@ -115,44 +125,6 @@ class MogileFS::Backend
|
|
115
125
|
add_command :set_state
|
116
126
|
add_command :replicate_now
|
117
127
|
|
118
|
-
# Errors copied from MogileFS/Worker/Query.pm
|
119
|
-
add_error 'dup'
|
120
|
-
add_error 'after_mismatch'
|
121
|
-
add_error 'bad_params'
|
122
|
-
add_error 'class_exists'
|
123
|
-
add_error 'class_has_files'
|
124
|
-
add_error 'class_not_found'
|
125
|
-
add_error 'db'
|
126
|
-
add_error 'domain_has_files'
|
127
|
-
add_error 'domain_exists'
|
128
|
-
add_error 'domain_not_empty'
|
129
|
-
add_error 'domain_not_found'
|
130
|
-
add_error 'failure'
|
131
|
-
add_error 'host_exists'
|
132
|
-
add_error 'host_mismatch'
|
133
|
-
add_error 'host_not_empty'
|
134
|
-
add_error 'host_not_found'
|
135
|
-
add_error 'invalid_chars'
|
136
|
-
add_error 'invalid_checker_level'
|
137
|
-
add_error 'invalid_mindevcount'
|
138
|
-
add_error 'key_exists'
|
139
|
-
add_error 'no_class'
|
140
|
-
add_error 'no_devices'
|
141
|
-
add_error 'no_domain'
|
142
|
-
add_error 'no_host'
|
143
|
-
add_error 'no_ip'
|
144
|
-
add_error 'no_key'
|
145
|
-
add_error 'no_port'
|
146
|
-
add_error 'none_match'
|
147
|
-
add_error 'plugin_aborted'
|
148
|
-
add_error 'state_too_high'
|
149
|
-
add_error 'size_verify_error'
|
150
|
-
add_error 'unknown_command'
|
151
|
-
add_error 'unknown_host'
|
152
|
-
add_error 'unknown_key'
|
153
|
-
add_error 'unknown_state'
|
154
|
-
add_error 'unreg_domain'
|
155
|
-
|
156
128
|
def shutdown_unlocked(do_raise = false) # :nodoc:
|
157
129
|
@pending = []
|
158
130
|
if @socket
|
@@ -254,11 +226,13 @@ class MogileFS::Backend
|
|
254
226
|
|
255
227
|
# Performs the +cmd+ request with +args+.
|
256
228
|
def do_request(cmd, args, idempotent = false)
|
257
|
-
|
229
|
+
no_raise = args.delete(:ruby_no_raise)
|
230
|
+
request = make_request(cmd, args)
|
258
231
|
@mutex.synchronize do
|
259
232
|
begin
|
260
233
|
io = dispatch_unlocked(request)
|
261
|
-
line = io.timed_gets(@timeout) and
|
234
|
+
line = io.timed_gets(@timeout) and
|
235
|
+
return parse_response(line, no_raise ? request : nil)
|
262
236
|
|
263
237
|
idempotent or
|
264
238
|
raise EOFError, "end of file reached after: #{request.inspect}"
|
@@ -349,17 +323,15 @@ class MogileFS::Backend
|
|
349
323
|
def socket
|
350
324
|
return @socket if @socket and not @socket.closed?
|
351
325
|
|
352
|
-
now = Time.now
|
353
|
-
|
354
326
|
@hosts.shuffle.each do |host|
|
355
|
-
next if
|
327
|
+
next if dead = @dead[host] and dead[0] > (Time.now - @fail_timeout)
|
356
328
|
|
357
329
|
begin
|
358
330
|
addr, port = host.split(/:/)
|
359
331
|
@socket = MogileFS::Socket.tcp(addr, port, @timeout)
|
360
332
|
@active_host = host
|
361
333
|
rescue SystemCallError, MogileFS::Timeout => err
|
362
|
-
@dead[host] = [ now, err ]
|
334
|
+
@dead[host] = [ Time.now, err ]
|
363
335
|
next
|
364
336
|
end
|
365
337
|
|
data/lib/mogilefs/chunker.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
|
-
require "digest/md5"
|
3
2
|
class MogileFS::Chunker
|
4
3
|
CRLF = "\r\n"
|
5
4
|
attr_reader :io
|
6
5
|
|
7
|
-
def initialize(io, md5)
|
6
|
+
def initialize(io, md5, expect_md5)
|
8
7
|
@io = io
|
9
|
-
@md5 = md5
|
8
|
+
@md5 = md5
|
9
|
+
@expect_md5 = expect_md5
|
10
10
|
end
|
11
11
|
|
12
12
|
def write(buf)
|
@@ -20,8 +20,14 @@ class MogileFS::Chunker
|
|
20
20
|
|
21
21
|
def flush
|
22
22
|
if @md5
|
23
|
-
content_md5 = [ @md5.digest ].pack('m').
|
24
|
-
|
23
|
+
content_md5 = [ @md5.digest ].pack('m').rstrip!
|
24
|
+
if @expect_md5.respond_to?(:call)
|
25
|
+
expect = @expect_md5.call.strip
|
26
|
+
if expect != content_md5
|
27
|
+
raise MogileFS::ChecksumMismatchError,
|
28
|
+
"expected: #{expect.inspect} actual: #{content_md5.inspect}"
|
29
|
+
end
|
30
|
+
end
|
25
31
|
@io.write("0\r\nContent-MD5: #{content_md5}\r\n\r\n")
|
26
32
|
else
|
27
33
|
@io.write("0\r\n\r\n")
|