mogilefs-client 2.2.0 → 3.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +11 -0
- data/.gemtest +0 -0
- data/.gitignore +4 -0
- data/.wrongdoc.yml +5 -0
- data/GIT-VERSION-GEN +28 -0
- data/GNUmakefile +44 -0
- data/HACKING +33 -0
- data/{History.txt → History} +0 -1
- data/{LICENSE.txt → LICENSE} +0 -1
- data/Manifest.txt +34 -7
- data/README +51 -0
- data/Rakefile +11 -11
- data/TODO +10 -0
- data/bin/mog +109 -68
- data/examples/mogstored_rack.rb +189 -0
- data/lib/mogilefs.rb +56 -17
- data/lib/mogilefs/admin.rb +128 -62
- data/lib/mogilefs/backend.rb +205 -95
- data/lib/mogilefs/bigfile.rb +54 -70
- data/lib/mogilefs/bigfile/filter.rb +58 -0
- data/lib/mogilefs/chunker.rb +30 -0
- data/lib/mogilefs/client.rb +0 -2
- data/lib/mogilefs/copy_stream.rb +30 -0
- data/lib/mogilefs/http_file.rb +175 -0
- data/lib/mogilefs/http_reader.rb +79 -0
- data/lib/mogilefs/mogilefs.rb +242 -148
- data/lib/mogilefs/mysql.rb +3 -4
- data/lib/mogilefs/paths_size.rb +24 -0
- data/lib/mogilefs/pool.rb +0 -1
- data/lib/mogilefs/socket.rb +9 -0
- data/lib/mogilefs/socket/kgio.rb +55 -0
- data/lib/mogilefs/socket/pure_ruby.rb +70 -0
- data/lib/mogilefs/socket_common.rb +58 -0
- data/lib/mogilefs/util.rb +6 -169
- data/test/aggregate.rb +11 -11
- data/test/exec.rb +72 -0
- data/test/fresh.rb +222 -0
- data/test/integration.rb +43 -0
- data/test/setup.rb +1 -0
- data/test/socket_test.rb +98 -0
- data/test/test_admin.rb +14 -37
- data/test/test_backend.rb +50 -107
- data/test/test_bigfile.rb +2 -2
- data/test/test_db_backend.rb +1 -2
- data/test/test_fresh.rb +8 -0
- data/test/test_http_reader.rb +34 -0
- data/test/test_mogilefs.rb +278 -98
- data/test/test_mogilefs_integration.rb +174 -0
- data/test/test_mogilefs_integration_large_pipe.rb +62 -0
- data/test/test_mogilefs_integration_list_keys.rb +40 -0
- data/test/test_mogilefs_socket_kgio.rb +11 -0
- data/test/test_mogilefs_socket_pure.rb +7 -0
- data/test/test_mogstored_rack.rb +89 -0
- data/test/test_mogtool_bigfile.rb +116 -0
- data/test/test_mysql.rb +1 -2
- data/test/test_pool.rb +1 -1
- data/test/test_unit_mogstored_rack.rb +72 -0
- metadata +76 -54
- data/README.txt +0 -80
- data/lib/mogilefs/httpfile.rb +0 -157
- data/lib/mogilefs/network.rb +0 -107
- data/test/test_network.rb +0 -56
- data/test/test_util.rb +0 -121
@@ -0,0 +1,174 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
require './test/integration'
|
3
|
+
|
4
|
+
class TestMogileFSIntegration < TestMogIntegration
|
5
|
+
def setup
|
6
|
+
super
|
7
|
+
@client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_CRUD
|
11
|
+
assert_equal 4, @client.store_content("CRUD", "default", "DATA")
|
12
|
+
assert_equal 4, @client.size("CRUD")
|
13
|
+
assert_equal "DATA", @client.get_file_data("CRUD")
|
14
|
+
assert_equal "DAT", @client.get_file_data("CRUD", nil, 3)
|
15
|
+
assert_equal "AT", @client.get_file_data("CRUD", nil, 2, 1)
|
16
|
+
|
17
|
+
tmp = Tempfile.new("z")
|
18
|
+
tmp_path = tmp.path
|
19
|
+
tmp.close!
|
20
|
+
assert_equal 4, @client.get_file_data("CRUD", tmp_path)
|
21
|
+
assert_equal "DATA", File.read(tmp_path)
|
22
|
+
st = File.stat(tmp_path)
|
23
|
+
assert_equal 0100600, st.mode
|
24
|
+
File.unlink(tmp_path)
|
25
|
+
|
26
|
+
sio = StringIO.new("")
|
27
|
+
rv = @client.get_file_data("CRUD", sio)
|
28
|
+
assert_equal 4, rv
|
29
|
+
assert_equal "DATA", sio.string
|
30
|
+
assert_equal 8, @client.store_content("CRUD", "default", "MOARDATA")
|
31
|
+
assert_equal "MOARDATA", @client.get_file_data("CRUD")
|
32
|
+
assert_equal true, @client.delete("CRUD")
|
33
|
+
assert_raises(MogileFS::Backend::UnknownKeyError) { @client.delete("CRUD") }
|
34
|
+
|
35
|
+
data = "hello world\n".freeze
|
36
|
+
tmp = tmpfile("blob")
|
37
|
+
tmp.sync = true
|
38
|
+
tmp.write(data)
|
39
|
+
tmp.rewind
|
40
|
+
assert_equal tmp.size, @client.store_file("blob", nil, tmp)
|
41
|
+
assert_equal(data, @client.get_file_data("blob"))
|
42
|
+
|
43
|
+
data = "pipe!\n".freeze
|
44
|
+
r, w = IO.pipe
|
45
|
+
th = Thread.new do
|
46
|
+
w.write(data)
|
47
|
+
w.close
|
48
|
+
end
|
49
|
+
assert_equal data.size, @client.store_file("pipe", nil, r)
|
50
|
+
assert_nothing_raised do
|
51
|
+
r.close
|
52
|
+
th.join
|
53
|
+
end
|
54
|
+
assert_equal(data, @client.get_file_data("pipe"))
|
55
|
+
|
56
|
+
cbk = MogileFS::Util::StoreContent.new(nil) do |write_callback|
|
57
|
+
10.times { write_callback.call("data") }
|
58
|
+
end
|
59
|
+
assert_nil cbk.length
|
60
|
+
nr = @client.store_content('store_content', nil, cbk)
|
61
|
+
assert_equal 40, nr
|
62
|
+
assert_equal("data" * 10, @client.get_file_data('store_content'))
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_store_non_rewindable
|
66
|
+
tmp = Object.new
|
67
|
+
def tmp.size
|
68
|
+
666
|
69
|
+
end
|
70
|
+
|
71
|
+
assert_raises(MogileFS::HTTPFile::NonRetryableError) do
|
72
|
+
@client.store_file("non_rewindable", nil, tmp)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_file_info
|
77
|
+
assert_equal 3, @client.store_content("file_info", "default", "FOO")
|
78
|
+
res = @client.file_info("file_info")
|
79
|
+
assert_kind_of Integer, res["fid"]
|
80
|
+
assert_equal 3, res["length"]
|
81
|
+
assert ! res.include?("devids")
|
82
|
+
assert_kind_of Integer, res["devcount"]
|
83
|
+
|
84
|
+
res = @client.file_info("file_info", :devices => true)
|
85
|
+
assert_kind_of Integer, res["fid"]
|
86
|
+
assert_equal 3, res["length"]
|
87
|
+
assert_kind_of Integer, res["devcount"]
|
88
|
+
devids = res.delete("devids")
|
89
|
+
assert_instance_of Array, devids
|
90
|
+
devids.each { |devid| assert_kind_of Integer, devid }
|
91
|
+
assert_equal res["devcount"], devids.size
|
92
|
+
end
|
93
|
+
|
94
|
+
def test_file_debug
|
95
|
+
assert_equal 3, @client.store_content("file_debug", "default", "BUG")
|
96
|
+
a = @client.file_debug("file_debug")
|
97
|
+
b = @client.file_debug(:key => "file_debug")
|
98
|
+
fid = @client.file_info("file_debug")["fid"]
|
99
|
+
c = @client.file_debug(fid)
|
100
|
+
d = @client.file_debug(:fid => fid)
|
101
|
+
|
102
|
+
[ a, b, c, d ].each do |res|
|
103
|
+
assert_equal fid, res["fid_fid"]
|
104
|
+
assert_equal 0, res["fid_classid"]
|
105
|
+
assert_equal "file_debug", res["fid_dkey"]
|
106
|
+
assert_equal 3, res["fid_length"]
|
107
|
+
assert_kind_of Array, res["devids"]
|
108
|
+
assert_kind_of Integer, res["devids"][0]
|
109
|
+
res["devids"].each do |devid|
|
110
|
+
uri = URI.parse(res["devpath_#{devid}"])
|
111
|
+
assert_equal "http", uri.scheme
|
112
|
+
end
|
113
|
+
assert_equal "default", res["fid_class"]
|
114
|
+
end
|
115
|
+
@client.delete("file_debug")
|
116
|
+
rv = @client.file_debug(fid)
|
117
|
+
assert rv.keys.grep(/\Afid_/).empty?, rv.inspect
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_file_debug_in_progress
|
121
|
+
rv = @client.new_file("file_debug_in_progress") do |http_file|
|
122
|
+
http_file << "ZZZZ"
|
123
|
+
dests = http_file.instance_variable_get(:@dests)
|
124
|
+
dests[0][1] =~ %r{/(\d+)\.fid\z}
|
125
|
+
fid = $1.to_i
|
126
|
+
rv = @client.file_debug(fid)
|
127
|
+
devids = dests.map { |x| x[0].to_i }.sort
|
128
|
+
assert_equal devids, rv["tempfile_devids"].sort
|
129
|
+
assert_equal "file_debug_in_progress", rv["tempfile_dkey"]
|
130
|
+
end
|
131
|
+
assert_equal 4, rv
|
132
|
+
end
|
133
|
+
|
134
|
+
def test_admin_get_devices
|
135
|
+
admin = MogileFS::Admin.new(:hosts => @trackers)
|
136
|
+
devices = admin.get_devices
|
137
|
+
if any_device = devices[0]
|
138
|
+
%w(mb_asof mb_free mb_used mb_total devid weight hostid).each do |field|
|
139
|
+
case value = any_device[field]
|
140
|
+
when nil
|
141
|
+
when Integer
|
142
|
+
assert value >= 0, "#{field}=#{value.inspect} is negative"
|
143
|
+
else
|
144
|
+
assert false, "#{field}=#{value.inspect} is #{value.class}"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
field = "utilization"
|
149
|
+
case value = any_device[field]
|
150
|
+
when nil
|
151
|
+
when Float
|
152
|
+
assert value >= 0.0, "#{field}=#{value.inspect} is negative"
|
153
|
+
else
|
154
|
+
assert false, "#{field}=#{value.inspect} is #{value.class}"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
# TODO: move this to a fresh instance
|
160
|
+
def test_admin_each_fid
|
161
|
+
admin = MogileFS::Admin.new(:hosts => @trackers)
|
162
|
+
seen = {}
|
163
|
+
count = admin.each_fid do |info|
|
164
|
+
seen[info["fid"]] = true
|
165
|
+
assert_kind_of Integer, info["fid"]
|
166
|
+
assert_kind_of Integer, info["length"]
|
167
|
+
assert_kind_of Integer, info["devcount"]
|
168
|
+
assert_kind_of String, info["key"]
|
169
|
+
assert_kind_of String, info["class"]
|
170
|
+
assert_kind_of String, info["domain"]
|
171
|
+
end
|
172
|
+
assert_equal count, seen.size
|
173
|
+
end if ENV["TEST_EXPENSIVE"]
|
174
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
require './test/integration'
|
3
|
+
require "digest/sha1"
|
4
|
+
|
5
|
+
class TestMogileFSLargePipe< TestMogIntegration
|
6
|
+
def setup
|
7
|
+
super
|
8
|
+
@client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
|
9
|
+
end
|
10
|
+
|
11
|
+
def test_large_pipe_test
|
12
|
+
junk = File.open("/dev/urandom") { |fp| fp.read(1024) }
|
13
|
+
junk *= 32
|
14
|
+
nr = rand(666) + 1024
|
15
|
+
r, w = IO.pipe
|
16
|
+
sha1 = Digest::SHA1.new
|
17
|
+
th = Thread.new do
|
18
|
+
nr.times do
|
19
|
+
sha1.update(junk)
|
20
|
+
w.write(junk)
|
21
|
+
end
|
22
|
+
w.close
|
23
|
+
end
|
24
|
+
assert_equal(nr * junk.size, @client.store_file("a", nil, r))
|
25
|
+
r.close
|
26
|
+
th.join
|
27
|
+
@client.get_file_data("a") do |rd|
|
28
|
+
assert_equal(nr * junk.size, @client.store_file("b", nil, rd))
|
29
|
+
end
|
30
|
+
a = Thread.new { @client.get_file_data("a") { |rd| sha1read(rd) } }
|
31
|
+
b = Thread.new { @client.get_file_data("b") { |rd| sha1read(rd) } }
|
32
|
+
a = a.value
|
33
|
+
b = b.value
|
34
|
+
assert_equal a, b
|
35
|
+
assert_equal sha1, a
|
36
|
+
|
37
|
+
# We should be able to open FIFOs
|
38
|
+
tmp = tmpfile("fifo")
|
39
|
+
tmp_path = tmp.path
|
40
|
+
File.unlink(tmp_path)
|
41
|
+
x!("mkfifo", tmp_path)
|
42
|
+
pid = fork do
|
43
|
+
File.open(tmp_path, "wb") do |wr|
|
44
|
+
nr.times { wr.write(junk) }
|
45
|
+
end
|
46
|
+
end
|
47
|
+
assert_equal(nr * junk.size, @client.store_file("fifo", nil, tmp_path))
|
48
|
+
_, status = Process.waitpid2(pid)
|
49
|
+
assert status.success?, status.inspect
|
50
|
+
fifo_sha1 = @client.get_file_data("fifo") { |rd| sha1read(rd) }
|
51
|
+
assert_equal sha1, fifo_sha1
|
52
|
+
end
|
53
|
+
|
54
|
+
def sha1read(rd)
|
55
|
+
buf = ""
|
56
|
+
d = Digest::SHA1.new
|
57
|
+
while rd.read(16384, buf)
|
58
|
+
d << buf
|
59
|
+
end
|
60
|
+
d
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
# -*- encoding: binary -*-
|
3
|
+
require './test/integration'
|
4
|
+
|
5
|
+
class TestMogileFSIntegrationListKeys < TestMogIntegration
|
6
|
+
def setup
|
7
|
+
super
|
8
|
+
@client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
|
9
|
+
end
|
10
|
+
|
11
|
+
def test_list_keys
|
12
|
+
k = %w(a b c d e f g)
|
13
|
+
k.each { |x| @client.store_content("lk_#{x}", nil, x) }
|
14
|
+
expect = k.map { |x| "lk_#{x}" }
|
15
|
+
rv = @client.list_keys
|
16
|
+
assert_equal([ expect, expect.last ] , rv)
|
17
|
+
nr = 0
|
18
|
+
@client.list_keys do |key, length, devcount|
|
19
|
+
assert_equal 1, length
|
20
|
+
assert_kind_of Integer, devcount
|
21
|
+
assert_equal expect[nr], key
|
22
|
+
nr += 1
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_list_keys_strange
|
27
|
+
@client.store_content("hello+world", nil, "HI")
|
28
|
+
rv = @client.list_keys
|
29
|
+
assert_equal "hello+world", rv[0][0]
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_each_key
|
33
|
+
9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) }
|
34
|
+
n = 0
|
35
|
+
@client.each_key do |key|
|
36
|
+
assert_equal "ek_#{n.to_s}", key
|
37
|
+
n += 1
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require "./test/socket_test"
|
2
|
+
require "mogilefs"
|
3
|
+
begin
|
4
|
+
require "kgio"
|
5
|
+
require "mogilefs/socket/kgio"
|
6
|
+
rescue LoadError
|
7
|
+
end unless ENV["MOGILEFS_CLIENT_PURE"]
|
8
|
+
|
9
|
+
class TestSocketKgio < Test::Unit::TestCase
|
10
|
+
include SocketTest
|
11
|
+
end if defined?(Kgio)
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
require "./test/fresh"
|
3
|
+
|
4
|
+
class TestMogstoredRack < Test::Unit::TestCase
|
5
|
+
include TestFreshSetup
|
6
|
+
def setup
|
7
|
+
setup_mogilefs
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_md5_check
|
11
|
+
add_host_device_domain
|
12
|
+
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
|
13
|
+
node = "#@test_host:#@mogstored_http_port"
|
14
|
+
pid = fork do
|
15
|
+
# not modifying this hash in the same process
|
16
|
+
MogileFS::HTTPFile::MD5_TRAILER_NODES[node] = true
|
17
|
+
client.store_content("md5_me", nil, "HELLO WORLD")
|
18
|
+
end
|
19
|
+
_, status = Process.waitpid2(pid)
|
20
|
+
assert status.success?, status.inspect
|
21
|
+
assert_equal "HELLO WORLD", client.get_file_data("md5_me")
|
22
|
+
end
|
23
|
+
|
24
|
+
def setup_mogstored
|
25
|
+
@docroot = Dir.mktmpdir(["mogfresh", "docroot"])
|
26
|
+
@mogstored_mgmt = TCPServer.new(@test_host, 0)
|
27
|
+
@mogstored_http = TCPServer.new(@test_host, 0)
|
28
|
+
@mogstored_mgmt_port = @mogstored_mgmt.addr[1]
|
29
|
+
@mogstored_http_port = @mogstored_http.addr[1]
|
30
|
+
@mogstored_conf = Tempfile.new(["mogstored", "conf"])
|
31
|
+
@mogstored_pid = Tempfile.new(["mogstored", "pid"])
|
32
|
+
@mogstored_conf.write <<EOF
|
33
|
+
pidfile = #{@mogstored_pid.path}
|
34
|
+
maxconns = 1000
|
35
|
+
mgmtlisten = #@test_host:#{@mogstored_mgmt_port}
|
36
|
+
server = none
|
37
|
+
docroot = #@docroot
|
38
|
+
EOF
|
39
|
+
@mogstored_conf.flush
|
40
|
+
@mogstored_mgmt.close
|
41
|
+
|
42
|
+
unicorn_setup
|
43
|
+
|
44
|
+
x!("mogstored", "--daemon", "--config=#{@mogstored_conf.path}")
|
45
|
+
wait_for_port @mogstored_mgmt_port
|
46
|
+
end
|
47
|
+
|
48
|
+
# I would use Rainbows! + *Threads + Ruby 1.9.3 in production
|
49
|
+
def unicorn_setup
|
50
|
+
examples_dir = Dir.pwd + "/examples"
|
51
|
+
assert File.directory?(examples_dir)
|
52
|
+
@ru = Tempfile.new(%w(mogstored_rack .ru))
|
53
|
+
@ru.write <<EOF
|
54
|
+
require "mogstored_rack"
|
55
|
+
use Rack::Head
|
56
|
+
run MogstoredRack.new("#@docroot")
|
57
|
+
EOF
|
58
|
+
@ru.flush
|
59
|
+
|
60
|
+
@unicorn_pid = Tempfile.new(%w(unicorn .pid))
|
61
|
+
@unicorn_conf = Tempfile.new(%w(unicorn.conf .rb))
|
62
|
+
@unicorn_stderr = Tempfile.new(%w(unicorn .stderr))
|
63
|
+
@unicorn_stdout = Tempfile.new(%w(unicorn .stdout))
|
64
|
+
@unicorn_conf.write <<EOF
|
65
|
+
listen "#@test_host:#{@mogstored_http_port}"
|
66
|
+
pid "#{@unicorn_pid.path}"
|
67
|
+
stderr_path "#{@unicorn_stderr.path}"
|
68
|
+
stdout_path "#{@unicorn_stdout.path}"
|
69
|
+
rewindable_input false
|
70
|
+
EOF
|
71
|
+
@unicorn_conf.flush
|
72
|
+
|
73
|
+
@mogstored_http.close
|
74
|
+
x!("unicorn", "-I", examples_dir, "-E", "deployment",
|
75
|
+
"--daemon", "--config", @unicorn_conf.path, @ru.path)
|
76
|
+
wait_for_port @mogstored_http_port
|
77
|
+
40.times do
|
78
|
+
break if File.size(@unicorn_pid.path) > 0
|
79
|
+
sleep 0.1
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def teardown
|
84
|
+
pid = File.read(@unicorn_pid.path).to_i
|
85
|
+
Process.kill(:QUIT, pid) if pid > 0
|
86
|
+
teardown_mogilefs
|
87
|
+
puts(@unicorn_stderr.read) if $DEBUG
|
88
|
+
end
|
89
|
+
end if `which unicorn`.chomp.size > 0
|
@@ -0,0 +1,116 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
require "./test/integration"
|
3
|
+
require "net/http"
|
4
|
+
ok = true
|
5
|
+
unless File.executable?(`which mogtool 2>/dev/null`.strip)
|
6
|
+
warn "mogtool not found, skipping #{__FILE__}"
|
7
|
+
ok = false
|
8
|
+
end
|
9
|
+
|
10
|
+
class TestMogtoolBigfile < TestMogIntegration
|
11
|
+
buf = File.open("/dev/urandom") { |fp| fp.read(1024) }
|
12
|
+
buf *= 1024
|
13
|
+
RAND = Tempfile.new("rand")
|
14
|
+
RAND.sync = true
|
15
|
+
sha1 = Digest::SHA1.new
|
16
|
+
100.times { sha1 << buf; RAND.write(buf) }
|
17
|
+
buf = nil
|
18
|
+
RAND_SHA1 = sha1.hexdigest
|
19
|
+
|
20
|
+
def setup
|
21
|
+
super
|
22
|
+
RAND.rewind
|
23
|
+
@big_uuid = "big-#{uuid}"
|
24
|
+
@client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
|
25
|
+
end
|
26
|
+
|
27
|
+
def mogtool!(*args)
|
28
|
+
x!("mogtool", "--trackers=#{@trackers.join(',')}",
|
29
|
+
"--domain=#@domain", *args)
|
30
|
+
end
|
31
|
+
|
32
|
+
# the mogtool definition of gzip is wrong and just raw zlib deflate
|
33
|
+
def test_bigfile_gzip_mogtool
|
34
|
+
mogtool!("inject", "--gzip", "--bigfile", RAND.path, @big_uuid)
|
35
|
+
sha1_check
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_bigfile_mogtool
|
39
|
+
mogtool!("inject", "--bigfile", RAND.path, @big_uuid)
|
40
|
+
sha1_check
|
41
|
+
|
42
|
+
# ensure fallback works for rebalanced/replaced files
|
43
|
+
part1 = "#@big_uuid,1"
|
44
|
+
tmp = tmpfile("part1")
|
45
|
+
before_uris = @client.get_uris(part1)
|
46
|
+
@client.get_file_data(part1, tmp)
|
47
|
+
@client.delete(part1)
|
48
|
+
@client.store_file(part1, nil, tmp.path)
|
49
|
+
wait_for_DELETE(before_uris)
|
50
|
+
sha1_check
|
51
|
+
|
52
|
+
# corrupt the existing data in part1
|
53
|
+
@client.store_content(part1, nil, "HELLO")
|
54
|
+
assert_nothing_raised { @client.get_uris(part1) }
|
55
|
+
|
56
|
+
# corruption is detected on verify
|
57
|
+
junk = tmpfile("junk")
|
58
|
+
assert_raises(MogileFS::ChecksumMismatchError) do
|
59
|
+
@client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
|
60
|
+
end
|
61
|
+
|
62
|
+
# corruption is NOT detected on verify
|
63
|
+
junk = tmpfile("junk")
|
64
|
+
assert_nothing_raised do
|
65
|
+
@client.bigfile_write("_big_info:#@big_uuid", junk, :verify => false)
|
66
|
+
end
|
67
|
+
|
68
|
+
# restoring no-corrupted data succeeds!
|
69
|
+
@client.store_file(part1, nil, tmp.path)
|
70
|
+
sha1_check
|
71
|
+
|
72
|
+
# missing parts fail
|
73
|
+
before_uris = @client.get_uris(part1)
|
74
|
+
@client.delete(part1)
|
75
|
+
junk = tmpfile("junk")
|
76
|
+
assert_raises(MogileFS::Backend::UnknownKeyError) do
|
77
|
+
@client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def wait_for_DELETE(uris)
|
82
|
+
uris.each do |uri|
|
83
|
+
tries = 0
|
84
|
+
begin
|
85
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
86
|
+
sleep(0.1) while Net::HTTPOK === http.head(uri.path)
|
87
|
+
end
|
88
|
+
rescue
|
89
|
+
if (tries += 1) < 666
|
90
|
+
sleep(0.1)
|
91
|
+
retry
|
92
|
+
end
|
93
|
+
raise
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def sha1_check
|
99
|
+
r, w = IO.pipe
|
100
|
+
@to_close << r
|
101
|
+
@to_close << w
|
102
|
+
th = Thread.new do
|
103
|
+
sha1 = Digest::SHA1.new
|
104
|
+
buf = ""
|
105
|
+
while r.read(16384, buf)
|
106
|
+
sha1 << buf
|
107
|
+
end
|
108
|
+
sha1.hexdigest
|
109
|
+
end
|
110
|
+
res = @client.bigfile_write("_big_info:#@big_uuid", w, :verify => true)
|
111
|
+
w.close
|
112
|
+
read_sha1 = th.value
|
113
|
+
assert_equal RAND_SHA1, read_sha1
|
114
|
+
assert_equal RAND.size, res[0]
|
115
|
+
end
|
116
|
+
end if ok
|