mogilefs-client 2.2.0 → 3.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +11 -0
- data/.gemtest +0 -0
- data/.gitignore +4 -0
- data/.wrongdoc.yml +5 -0
- data/GIT-VERSION-GEN +28 -0
- data/GNUmakefile +44 -0
- data/HACKING +33 -0
- data/{History.txt → History} +0 -1
- data/{LICENSE.txt → LICENSE} +0 -1
- data/Manifest.txt +34 -7
- data/README +51 -0
- data/Rakefile +11 -11
- data/TODO +10 -0
- data/bin/mog +109 -68
- data/examples/mogstored_rack.rb +189 -0
- data/lib/mogilefs.rb +56 -17
- data/lib/mogilefs/admin.rb +128 -62
- data/lib/mogilefs/backend.rb +205 -95
- data/lib/mogilefs/bigfile.rb +54 -70
- data/lib/mogilefs/bigfile/filter.rb +58 -0
- data/lib/mogilefs/chunker.rb +30 -0
- data/lib/mogilefs/client.rb +0 -2
- data/lib/mogilefs/copy_stream.rb +30 -0
- data/lib/mogilefs/http_file.rb +175 -0
- data/lib/mogilefs/http_reader.rb +79 -0
- data/lib/mogilefs/mogilefs.rb +242 -148
- data/lib/mogilefs/mysql.rb +3 -4
- data/lib/mogilefs/paths_size.rb +24 -0
- data/lib/mogilefs/pool.rb +0 -1
- data/lib/mogilefs/socket.rb +9 -0
- data/lib/mogilefs/socket/kgio.rb +55 -0
- data/lib/mogilefs/socket/pure_ruby.rb +70 -0
- data/lib/mogilefs/socket_common.rb +58 -0
- data/lib/mogilefs/util.rb +6 -169
- data/test/aggregate.rb +11 -11
- data/test/exec.rb +72 -0
- data/test/fresh.rb +222 -0
- data/test/integration.rb +43 -0
- data/test/setup.rb +1 -0
- data/test/socket_test.rb +98 -0
- data/test/test_admin.rb +14 -37
- data/test/test_backend.rb +50 -107
- data/test/test_bigfile.rb +2 -2
- data/test/test_db_backend.rb +1 -2
- data/test/test_fresh.rb +8 -0
- data/test/test_http_reader.rb +34 -0
- data/test/test_mogilefs.rb +278 -98
- data/test/test_mogilefs_integration.rb +174 -0
- data/test/test_mogilefs_integration_large_pipe.rb +62 -0
- data/test/test_mogilefs_integration_list_keys.rb +40 -0
- data/test/test_mogilefs_socket_kgio.rb +11 -0
- data/test/test_mogilefs_socket_pure.rb +7 -0
- data/test/test_mogstored_rack.rb +89 -0
- data/test/test_mogtool_bigfile.rb +116 -0
- data/test/test_mysql.rb +1 -2
- data/test/test_pool.rb +1 -1
- data/test/test_unit_mogstored_rack.rb +72 -0
- metadata +76 -54
- data/README.txt +0 -80
- data/lib/mogilefs/httpfile.rb +0 -157
- data/lib/mogilefs/network.rb +0 -107
- data/test/test_network.rb +0 -56
- data/test/test_util.rb +0 -121
data/test/test_backend.rb
CHANGED
@@ -1,11 +1,9 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
require 'test/unit'
|
3
|
-
require 'test/setup'
|
3
|
+
require './test/setup'
|
4
4
|
|
5
5
|
$TESTING = true
|
6
6
|
|
7
|
-
require 'mogilefs/backend'
|
8
|
-
|
9
7
|
class MogileFS::Backend
|
10
8
|
|
11
9
|
attr_accessor :hosts
|
@@ -36,37 +34,18 @@ class TestBackend < Test::Unit::TestCase
|
|
36
34
|
end
|
37
35
|
|
38
36
|
def test_do_request
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
client
|
43
|
-
|
44
|
-
client
|
45
|
-
end
|
46
|
-
|
47
|
-
@backend.hosts = [ "127.0.0.1:#{tmp.port}" ]
|
48
|
-
|
37
|
+
srv = TCPServer.new("127.0.0.1", 0)
|
38
|
+
port = srv.addr[1]
|
39
|
+
accepted = Thread.new do
|
40
|
+
client = srv.accept
|
41
|
+
client.write("OK 1 you=win\r\n")
|
42
|
+
client
|
43
|
+
end
|
44
|
+
@backend.hosts = [ "127.0.0.1:#{port}" ]
|
49
45
|
assert_equal({'you' => 'win'},
|
50
46
|
@backend.do_request('go!', { 'fight' => 'team fight!' }))
|
51
|
-
|
52
|
-
assert_equal "go! fight=team+fight%21\r\n",
|
53
|
-
ensure
|
54
|
-
TempServer.destroy_all!
|
55
|
-
end
|
56
|
-
|
57
|
-
def test_do_request_send_error
|
58
|
-
socket_request = ''
|
59
|
-
socket = Object.new
|
60
|
-
def socket.closed?() false end
|
61
|
-
def socket.write(request) raise SystemCallError, 'dummy' end
|
62
|
-
|
63
|
-
@backend.instance_variable_set '@socket', socket
|
64
|
-
|
65
|
-
assert_raises MogileFS::UnreachableBackendError do
|
66
|
-
@backend.do_request 'go!', { 'fight' => 'team fight!' }
|
67
|
-
end
|
68
|
-
|
69
|
-
assert_equal nil, @backend.instance_variable_get('@socket')
|
47
|
+
accepted = accepted.value
|
48
|
+
assert_equal "go! fight=team+fight%21\r\n", accepted.readpartial(4096)
|
70
49
|
end
|
71
50
|
|
72
51
|
def test_automatic_exception
|
@@ -89,19 +68,6 @@ class TestBackend < Test::Unit::TestCase
|
|
89
68
|
assert MogileFS::Backend.const_defined?('SizeVerifyError')
|
90
69
|
end
|
91
70
|
|
92
|
-
def test_do_request_truncated
|
93
|
-
socket_request = ''
|
94
|
-
socket = Object.new
|
95
|
-
def socket.closed?() false end
|
96
|
-
def socket.write(request) return request.length - 1 end
|
97
|
-
|
98
|
-
@backend.instance_variable_set '@socket', socket
|
99
|
-
|
100
|
-
assert_raises MogileFS::RequestTruncatedError do
|
101
|
-
@backend.do_request 'go!', { 'fight' => 'team fight!' }
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
71
|
def test_make_request
|
106
72
|
assert_equal "go! fight=team+fight%21\r\n",
|
107
73
|
@backend.make_request('go!', { 'fight' => 'team fight!' })
|
@@ -109,11 +75,11 @@ class TestBackend < Test::Unit::TestCase
|
|
109
75
|
|
110
76
|
def test_parse_response
|
111
77
|
assert_equal({'foo' => 'bar', 'baz' => 'hoge'},
|
112
|
-
@backend.parse_response(
|
78
|
+
@backend.parse_response("OK 1 foo=bar&baz=hoge\r\n"))
|
113
79
|
|
114
80
|
err = nil
|
115
81
|
begin
|
116
|
-
@backend.parse_response(
|
82
|
+
@backend.parse_response("ERR you totally suck\r\n")
|
117
83
|
rescue MogileFS::Error => err
|
118
84
|
assert_equal 'MogileFS::Backend::YouError', err.class.to_s
|
119
85
|
assert_equal 'totally suck', err.message
|
@@ -126,6 +92,9 @@ class TestBackend < Test::Unit::TestCase
|
|
126
92
|
assert_raises MogileFS::InvalidResponseError do
|
127
93
|
@backend.parse_response 'garbage'
|
128
94
|
end
|
95
|
+
assert_raises MogileFS::InvalidResponseError do
|
96
|
+
@backend.parse_response("OK 1 foo=bar&baz=hoge")
|
97
|
+
end
|
129
98
|
end
|
130
99
|
|
131
100
|
def test_parse_response_newline
|
@@ -141,89 +110,64 @@ class TestBackend < Test::Unit::TestCase
|
|
141
110
|
end
|
142
111
|
|
143
112
|
def test_readable_eh_not_readable
|
144
|
-
|
145
|
-
|
113
|
+
srv = TCPServer.new("127.0.0.1", 0)
|
114
|
+
port = srv.addr[1]
|
115
|
+
@backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{port}" ],
|
146
116
|
:timeout => 0.5)
|
147
117
|
begin
|
148
118
|
@backend.do_request 'foo', {}
|
149
119
|
rescue MogileFS::UnreadableSocketError => e
|
150
|
-
assert_equal "127.0.0.1:#{
|
120
|
+
assert_equal "127.0.0.1:#{port} never became readable", e.message
|
151
121
|
rescue Exception => err
|
152
122
|
flunk "MogileFS::UnreadableSocketError not raised #{err} #{err.backtrace}"
|
153
123
|
else
|
154
124
|
flunk "MogileFS::UnreadableSocketError not raised"
|
155
|
-
ensure
|
156
|
-
TempServer.destroy_all!
|
157
125
|
end
|
158
126
|
end
|
159
127
|
|
160
|
-
def
|
128
|
+
def test_socket_dead
|
161
129
|
assert_equal({}, @backend.dead)
|
162
|
-
assert_raises
|
130
|
+
assert_raises(MogileFS::UnreachableBackendError) do
|
131
|
+
@backend.do_request('test', {})
|
132
|
+
end
|
163
133
|
assert_equal(['localhost:1'], @backend.dead.keys)
|
164
134
|
end
|
165
135
|
|
166
|
-
def
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
sleep
|
179
|
-
end
|
180
|
-
nr = 10
|
181
|
-
|
182
|
-
nr.times do
|
183
|
-
begin
|
184
|
-
t1 = TempServer.new(bad, ENV['TEST_DEAD_PORT'])
|
185
|
-
t2 = TempServer.new(good)
|
186
|
-
hosts = ["127.0.0.1:#{t1.port}", "127.0.0.1:#{t2.port}"]
|
187
|
-
@backend = MogileFS::Backend.new(:hosts => hosts.dup)
|
188
|
-
assert_equal({}, @backend.dead)
|
189
|
-
old_chld_handler = trap('CHLD', 'DEFAULT')
|
190
|
-
t1.destroy!
|
191
|
-
Process.waitpid(t1.pid)
|
192
|
-
trap('CHLD', old_chld_handler)
|
193
|
-
sock = @backend.socket
|
194
|
-
assert_equal Socket, sock.class
|
195
|
-
port = Socket.unpack_sockaddr_in(sock.getpeername).first
|
196
|
-
# p [ 'ports', "port=#{port}", "t1=#{t1.port}", "t2=#{t2.port}" ]
|
197
|
-
assert_equal t2.port, port
|
198
|
-
IO.select([sock])
|
199
|
-
assert_equal '.', sock.sysread(1)
|
200
|
-
ensure
|
201
|
-
TempServer.destroy_all!
|
136
|
+
def test_socket_robust_on_dead_server
|
137
|
+
10.times do
|
138
|
+
t1 = TCPServer.new("127.0.0.1", 0)
|
139
|
+
t2 = TCPServer.new("127.0.0.1", 0)
|
140
|
+
hosts = ["127.0.0.1:#{t1.addr[1]}", "127.0.0.1:#{t2.addr[1]}"]
|
141
|
+
@backend = MogileFS::Backend.new(:hosts => hosts.dup)
|
142
|
+
assert_equal({}, @backend.dead)
|
143
|
+
t1.close
|
144
|
+
thr = Thread.new do
|
145
|
+
client = t2.accept
|
146
|
+
client.write("OK 1 foo=bar\r\n")
|
147
|
+
client
|
202
148
|
end
|
203
|
-
|
204
|
-
|
205
|
-
|
149
|
+
rv = nil
|
150
|
+
assert_nothing_raised do
|
151
|
+
rv = @backend.do_request('test', { "all" => "ALL" })
|
152
|
+
end
|
153
|
+
accepted = thr.value
|
154
|
+
assert_equal "test all=ALL\r\n", accepted.readpartial(666)
|
155
|
+
assert_equal({"foo"=>"bar"}, rv)
|
156
|
+
end
|
206
157
|
end
|
207
158
|
|
208
159
|
def test_shutdown
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
accept_nr += 1
|
213
|
-
r = IO.select([client], [client])
|
214
|
-
client.syswrite(accept_nr.to_s)
|
215
|
-
sleep
|
216
|
-
end)
|
217
|
-
@backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
|
160
|
+
srv = TCPServer.new('127.0.0.1', 0)
|
161
|
+
port = srv.addr[1]
|
162
|
+
@backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{port}" ]
|
218
163
|
assert @backend.socket
|
219
164
|
assert ! @backend.socket.closed?
|
220
|
-
|
221
|
-
|
165
|
+
client = srv.accept
|
166
|
+
client.write '1'
|
167
|
+
resp = @backend.socket.read(1)
|
222
168
|
@backend.shutdown
|
223
169
|
assert_equal nil, @backend.instance_variable_get(:@socket)
|
224
170
|
assert_equal 1, resp.to_i
|
225
|
-
ensure
|
226
|
-
TempServer.destroy_all!
|
227
171
|
end
|
228
172
|
|
229
173
|
def test_url_decode
|
@@ -273,7 +217,6 @@ class TestBackend < Test::Unit::TestCase
|
|
273
217
|
actual = input.map { |c| @backend.url_unescape c }.join "\n"
|
274
218
|
|
275
219
|
expected = (0..255).map { |c| c.chr }.join "\n"
|
276
|
-
expected.sub! '+', ' '
|
277
220
|
|
278
221
|
assert_equal expected, actual
|
279
222
|
end
|
data/test/test_bigfile.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
|
-
require 'test/setup'
|
2
|
+
require './test/setup'
|
3
3
|
require 'mogilefs/bigfile'
|
4
4
|
|
5
5
|
class TestMogileFS__Bigfile < TestMogileFS
|
@@ -41,7 +41,7 @@ size 18
|
|
41
41
|
part 1 bytes=12 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo1:7500/dev2/0/000/144/0000144411.fid, http://foo2:7500/dev1/0/000/144/0000144411.fid
|
42
42
|
part 2 bytes=6 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo4:7500/dev2/0/000/144/0000144411.fid, http://foo3:7500/dev1/0/000/144/0000144411.fid
|
43
43
|
EOS
|
44
|
-
i =
|
44
|
+
i = bigfile_parse_info(s)
|
45
45
|
assert_equal expect, i
|
46
46
|
end
|
47
47
|
|
data/test/test_db_backend.rb
CHANGED
data/test/test_fresh.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
require "./test/setup"
|
3
|
+
|
4
|
+
class TestHTTPReader < Test::Unit::TestCase
|
5
|
+
def rsock
|
6
|
+
host = "127.0.0.1"
|
7
|
+
s = TCPServer.new(host, 0)
|
8
|
+
th = Thread.new do
|
9
|
+
c = s.accept
|
10
|
+
c.readpartial(0x666)
|
11
|
+
c.write("HTTP/1.0 200 OK\r\nContent-Length: 666\r\n\r\n666")
|
12
|
+
c.close
|
13
|
+
end
|
14
|
+
path = "http://#{host}:#{s.addr[1]}/"
|
15
|
+
r = MogileFS::HTTPReader.try(path, 666, nil)
|
16
|
+
assert_nil th.value
|
17
|
+
assert_kind_of IO, r
|
18
|
+
r
|
19
|
+
ensure
|
20
|
+
s.close
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_short_to_s
|
24
|
+
r = rsock
|
25
|
+
assert_raises(MogileFS::SizeMismatchError) { r.to_s }
|
26
|
+
r.close
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_short_stream_to
|
30
|
+
r = rsock
|
31
|
+
assert_raises(MogileFS::SizeMismatchError) { r.stream_to("/dev/null") }
|
32
|
+
r.close
|
33
|
+
end
|
34
|
+
end
|
data/test/test_mogilefs.rb
CHANGED
@@ -1,12 +1,10 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
|
-
require 'test/setup'
|
2
|
+
require './test/setup'
|
3
3
|
require 'stringio'
|
4
4
|
require 'tempfile'
|
5
5
|
require 'fileutils'
|
6
6
|
|
7
7
|
class TestMogileFS__MogileFS < TestMogileFS
|
8
|
-
include MogileFS::Util
|
9
|
-
|
10
8
|
def setup
|
11
9
|
@klass = MogileFS::MogileFS
|
12
10
|
super
|
@@ -24,7 +22,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
24
22
|
tmp = Tempfile.new('accept')
|
25
23
|
accept = File.open(tmp.path, "ab")
|
26
24
|
svr = Proc.new do |serv, port|
|
27
|
-
client,
|
25
|
+
client, _ = serv.accept
|
28
26
|
client.sync = true
|
29
27
|
readed = client.recv(4096, 0)
|
30
28
|
assert(readed =~ \
|
@@ -50,7 +48,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
50
48
|
tmp = Tempfile.new('accept')
|
51
49
|
accept = File.open(tmp.path, 'ab')
|
52
50
|
svr1 = Proc.new do |serv, port|
|
53
|
-
client,
|
51
|
+
client, _ = serv.accept
|
54
52
|
client.sync = true
|
55
53
|
readed = client.recv(4096, 0)
|
56
54
|
assert(readed =~ \
|
@@ -61,7 +59,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
61
59
|
end
|
62
60
|
|
63
61
|
svr2 = Proc.new do |serv, port|
|
64
|
-
client,
|
62
|
+
client, _ = serv.accept
|
65
63
|
client.sync = true
|
66
64
|
readed = client.recv(4096, 0)
|
67
65
|
assert(readed =~ \
|
@@ -97,13 +95,13 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
97
95
|
|
98
96
|
accept = Tempfile.new('accept')
|
99
97
|
svr = Proc.new do |serv, port|
|
100
|
-
client,
|
98
|
+
client, _ = serv.accept
|
101
99
|
client.sync = true
|
102
100
|
accept.syswrite('.')
|
103
101
|
readed = client.recv(4096, 0)
|
104
102
|
assert(readed =~ \
|
105
103
|
%r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
|
106
|
-
|
104
|
+
MogileFS.io.copy_stream(tmpfp, client)
|
107
105
|
client.close
|
108
106
|
exit 0
|
109
107
|
end
|
@@ -210,40 +208,6 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
210
208
|
assert_equal 'new_key_2', next_after
|
211
209
|
end
|
212
210
|
|
213
|
-
def test_list_keys_block
|
214
|
-
@backend.list_keys = { 'key_count' => '2', 'next_after' => 'new_key_2',
|
215
|
-
'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
|
216
|
-
http_resp = "HTTP/1.0 200 OK\r\nContent-Length: %u\r\n"
|
217
|
-
srv = Proc.new do |serv, port, size|
|
218
|
-
client, client_addr = serv.accept
|
219
|
-
client.sync = true
|
220
|
-
readed = client.readpartial(4096)
|
221
|
-
assert %r{\AHEAD } =~ readed
|
222
|
-
client.send(http_resp % size, 0)
|
223
|
-
client.close
|
224
|
-
end
|
225
|
-
t1 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
|
226
|
-
t2 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
|
227
|
-
t3 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 10) })
|
228
|
-
@backend.get_paths = { 'paths' => '2',
|
229
|
-
'path1' => "http://127.0.0.1:#{t1.port}/",
|
230
|
-
'path2' => "http://127.0.0.1:#{t2.port}/" }
|
231
|
-
@backend.get_paths = { 'paths' => '1',
|
232
|
-
'path1' => "http://127.0.0.1:#{t3.port}/" }
|
233
|
-
|
234
|
-
res = []
|
235
|
-
keys, next_after = @client.list_keys('new') do |key,length,devcount|
|
236
|
-
res << [ key, length, devcount ]
|
237
|
-
end
|
238
|
-
|
239
|
-
expect_res = [ [ 'new_key_1', 5, 2 ], [ 'new_key_2', 10, 1 ] ]
|
240
|
-
assert_equal expect_res, res
|
241
|
-
assert_equal ['new_key_1', 'new_key_2'], keys.sort
|
242
|
-
assert_equal 'new_key_2', next_after
|
243
|
-
ensure
|
244
|
-
TempServer.destroy_all!
|
245
|
-
end
|
246
|
-
|
247
211
|
def test_new_file_http
|
248
212
|
@client.readonly = true
|
249
213
|
assert_raises MogileFS::ReadOnlyError do
|
@@ -258,44 +222,6 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
258
222
|
end
|
259
223
|
end
|
260
224
|
|
261
|
-
def test_size_http
|
262
|
-
accept = Tempfile.new('accept')
|
263
|
-
t = TempServer.new(Proc.new do |serv,port|
|
264
|
-
client, client_addr = serv.accept
|
265
|
-
client.sync = true
|
266
|
-
readed = client.recv(4096, 0) rescue nil
|
267
|
-
accept.syswrite('.')
|
268
|
-
assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
|
269
|
-
client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\n", 0)
|
270
|
-
client.close
|
271
|
-
end)
|
272
|
-
|
273
|
-
path = "http://127.0.0.1:#{t.port}/path"
|
274
|
-
@backend.get_paths = { 'paths' => 1, 'path1' => path }
|
275
|
-
|
276
|
-
assert_equal 5, @client.size('key')
|
277
|
-
assert_equal 1, accept.stat.size
|
278
|
-
end
|
279
|
-
|
280
|
-
def test_bad_size_http
|
281
|
-
tmp = Tempfile.new('accept')
|
282
|
-
t = TempServer.new(Proc.new do |serv,port|
|
283
|
-
client, client_addr = serv.accept
|
284
|
-
client.sync = true
|
285
|
-
readed = client.recv(4096, 0) rescue nil
|
286
|
-
assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
|
287
|
-
tmp.syswrite('.')
|
288
|
-
client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
|
289
|
-
client.close
|
290
|
-
end)
|
291
|
-
|
292
|
-
path = "http://127.0.0.1:#{t.port}/path"
|
293
|
-
@backend.get_paths = { 'paths' => 1, 'path1' => path }
|
294
|
-
|
295
|
-
assert_nil @client.size('key')
|
296
|
-
assert_equal 1, tmp.stat.size
|
297
|
-
end
|
298
|
-
|
299
225
|
def test_store_file_small_http
|
300
226
|
received = Tempfile.new('received')
|
301
227
|
to_store = Tempfile.new('small')
|
@@ -303,9 +229,9 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
303
229
|
|
304
230
|
expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
|
305
231
|
t = TempServer.new(Proc.new do |serv, accept|
|
306
|
-
client,
|
232
|
+
client, _ = serv.accept
|
307
233
|
client.sync = true
|
308
|
-
received.syswrite(client.
|
234
|
+
received.syswrite(client.read(expected.bytesize))
|
309
235
|
client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
|
310
236
|
client.close
|
311
237
|
end)
|
@@ -327,9 +253,14 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
327
253
|
expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
|
328
254
|
|
329
255
|
t = TempServer.new(Proc.new do |serv, accept|
|
330
|
-
client,
|
256
|
+
client, _ = serv.accept
|
331
257
|
client.sync = true
|
332
|
-
|
258
|
+
seen = ""
|
259
|
+
while seen !~ /\r\n\r\ndata/
|
260
|
+
buf = client.readpartial(4096)
|
261
|
+
seen << buf
|
262
|
+
received.syswrite(buf)
|
263
|
+
end
|
333
264
|
client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
|
334
265
|
client.close
|
335
266
|
end)
|
@@ -357,7 +288,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
357
288
|
expected += "data"
|
358
289
|
end
|
359
290
|
t = TempServer.new(Proc.new do |serv, accept|
|
360
|
-
client,
|
291
|
+
client, _ = serv.accept
|
361
292
|
client.sync = true
|
362
293
|
nr = 0
|
363
294
|
loop do
|
@@ -391,23 +322,35 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
391
322
|
TempServer.destroy_all!
|
392
323
|
end
|
393
324
|
|
394
|
-
def
|
325
|
+
def test_store_content_multi_dest_failover_path
|
326
|
+
test_store_content_multi_dest_failover(true)
|
327
|
+
end
|
328
|
+
|
329
|
+
def test_store_content_multi_dest_failover(big_io = false)
|
395
330
|
received1 = Tempfile.new('received')
|
396
331
|
received2 = Tempfile.new('received')
|
397
332
|
expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
|
398
333
|
|
399
334
|
t1 = TempServer.new(Proc.new do |serv, accept|
|
400
|
-
client,
|
401
|
-
|
402
|
-
|
335
|
+
client, _ = serv.accept
|
336
|
+
seen = ""
|
337
|
+
while seen !~ /\r\n\r\ndata/
|
338
|
+
buf = client.readpartial(4096)
|
339
|
+
seen << buf
|
340
|
+
received1.syswrite(buf)
|
341
|
+
end
|
403
342
|
client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
|
404
343
|
client.close
|
405
344
|
end)
|
406
345
|
|
407
346
|
t2 = TempServer.new(Proc.new do |serv, accept|
|
408
|
-
client,
|
409
|
-
|
410
|
-
|
347
|
+
client, _ = serv.accept
|
348
|
+
seen = ""
|
349
|
+
while seen !~ /\r\n\r\ndata/
|
350
|
+
buf = client.readpartial(4096)
|
351
|
+
seen << buf
|
352
|
+
received2.syswrite(buf)
|
353
|
+
end
|
411
354
|
client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
|
412
355
|
client.close
|
413
356
|
end)
|
@@ -420,7 +363,15 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
420
363
|
'path_2' => "http://127.0.0.1:#{t2.port}/path",
|
421
364
|
}
|
422
365
|
|
423
|
-
|
366
|
+
if big_io
|
367
|
+
tmp = Tempfile.new('data')
|
368
|
+
tmp.sync = true
|
369
|
+
tmp.write 'data'
|
370
|
+
nr = @client.store_file('new_key', 'test', tmp.path)
|
371
|
+
tmp.close!
|
372
|
+
else
|
373
|
+
nr = @client.store_content 'new_key', 'test', 'data'
|
374
|
+
end
|
424
375
|
assert_equal 4, nr
|
425
376
|
received1.sysseek(0)
|
426
377
|
received2.sysseek(0)
|
@@ -432,7 +383,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
432
383
|
|
433
384
|
def test_store_content_http_fail
|
434
385
|
t = TempServer.new(Proc.new do |serv, accept|
|
435
|
-
client,
|
386
|
+
client, _ = serv.accept
|
436
387
|
client.sync = true
|
437
388
|
client.recv(4096, 0)
|
438
389
|
client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
|
@@ -444,7 +395,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
444
395
|
'path' => "http://127.0.0.1:#{t.port}/path",
|
445
396
|
}
|
446
397
|
|
447
|
-
assert_raises MogileFS::HTTPFile::
|
398
|
+
assert_raises MogileFS::HTTPFile::NoStorageNodesError do
|
448
399
|
@client.store_content 'new_key', 'test', 'data'
|
449
400
|
end
|
450
401
|
end
|
@@ -453,7 +404,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
453
404
|
received = Tempfile.new('received')
|
454
405
|
expected = "PUT /path HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
|
455
406
|
t = TempServer.new(Proc.new do |serv, accept|
|
456
|
-
client,
|
407
|
+
client, _ = serv.accept
|
457
408
|
client.sync = true
|
458
409
|
received.syswrite(client.recv(4096, 0))
|
459
410
|
client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
|
@@ -503,7 +454,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
503
454
|
|
504
455
|
readed = Tempfile.new('readed')
|
505
456
|
t = TempServer.new(Proc.new do |serv, accept|
|
506
|
-
client,
|
457
|
+
client, _ = serv.accept
|
507
458
|
client.sync = true
|
508
459
|
nr = 0
|
509
460
|
loop do
|
@@ -525,7 +476,7 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
525
476
|
|
526
477
|
orig_size = to_put.size
|
527
478
|
nr = @client.store_file('new_key', 'test', to_put.path)
|
528
|
-
assert nr
|
479
|
+
assert nr, nr.inspect
|
529
480
|
assert_equal orig_size, nr
|
530
481
|
assert_equal orig_size, to_put.size
|
531
482
|
readed.sysseek(0)
|
@@ -592,6 +543,235 @@ class TestMogileFS__MogileFS < TestMogileFS
|
|
592
543
|
assert_equal 'readonly mogilefs', e.message
|
593
544
|
end
|
594
545
|
|
546
|
+
def assert_get_paths_args(expect, *args)
|
547
|
+
sock = TCPServer.new("127.0.0.1", 0)
|
548
|
+
nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
|
549
|
+
c = MogileFS::MogileFS.new(nargs)
|
550
|
+
received = []
|
551
|
+
th = Thread.new do
|
552
|
+
a = sock.accept
|
553
|
+
line = a.gets
|
554
|
+
received << line
|
555
|
+
a.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
|
556
|
+
a.close
|
557
|
+
end
|
558
|
+
paths_expect = %w(http://0/a http://0/b)
|
559
|
+
assert_equal paths_expect, c.get_paths("f", *args)
|
560
|
+
th.join
|
561
|
+
assert_equal 1, received.size
|
562
|
+
tmp = c.backend.url_decode(received[0].split(/\s+/)[1])
|
563
|
+
assert_equal "f", tmp.delete("key")
|
564
|
+
assert_equal "foo", tmp.delete("domain")
|
565
|
+
assert_equal expect, tmp
|
566
|
+
c.backend.shutdown
|
567
|
+
ensure
|
568
|
+
sock.close
|
569
|
+
end
|
570
|
+
|
571
|
+
def test_get_paths_args
|
572
|
+
assert_get_paths_args({"noverify"=>"1", "zone"=>""})
|
573
|
+
assert_get_paths_args({"noverify"=>"0", "zone"=>""}, false)
|
574
|
+
assert_get_paths_args({"noverify"=>"0", "zone"=>""}, :noverify=>false)
|
575
|
+
assert_get_paths_args({"noverify"=>"1", "zone"=>"alt"}, true, "alt")
|
576
|
+
assert_get_paths_args({"noverify"=>"1", "zone"=>"alt"},
|
577
|
+
{:noverify => true, :zone => "alt"})
|
578
|
+
assert_get_paths_args({"noverify"=>"1", "zone"=>"alt","pathcount"=>"666"},
|
579
|
+
{:noverify => true, :zone => "alt", :pathcount=>666})
|
580
|
+
end
|
581
|
+
|
582
|
+
def test_idempotent_command_eof
|
583
|
+
ip = "127.0.0.1"
|
584
|
+
a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
|
585
|
+
hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
|
586
|
+
args = { :hosts => hosts, :domain => "foo" }
|
587
|
+
c = MogileFS::MogileFS.new(args)
|
588
|
+
received = []
|
589
|
+
th = Thread.new do
|
590
|
+
r = IO.select([a, b])
|
591
|
+
x = r[0][0].accept
|
592
|
+
received << x.gets
|
593
|
+
x.close
|
594
|
+
|
595
|
+
r = IO.select([a, b])
|
596
|
+
x = r[0][0].accept
|
597
|
+
received << x.gets
|
598
|
+
x.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
|
599
|
+
x.close
|
600
|
+
end
|
601
|
+
expect = %w(http://0/a http://0/b)
|
602
|
+
assert_equal expect, c.get_paths("f")
|
603
|
+
th.join
|
604
|
+
assert_equal 2, received.size
|
605
|
+
assert_equal received[0], received[1]
|
606
|
+
end
|
607
|
+
|
608
|
+
def test_idempotent_command_response_truncated
|
609
|
+
ip = "127.0.0.1"
|
610
|
+
a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
|
611
|
+
hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
|
612
|
+
args = { :hosts => hosts, :domain => "foo" }
|
613
|
+
c = MogileFS::MogileFS.new(args)
|
614
|
+
received = []
|
615
|
+
th = Thread.new do
|
616
|
+
r = IO.select([a, b])
|
617
|
+
x = r[0][0].accept
|
618
|
+
received << x.gets
|
619
|
+
x.write("OK paths=2&path1=http://0/a&path2=http://0/")
|
620
|
+
x.close
|
621
|
+
|
622
|
+
r = IO.select([a, b])
|
623
|
+
x = r[0][0].accept
|
624
|
+
received << x.gets
|
625
|
+
x.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
|
626
|
+
x.close
|
627
|
+
end
|
628
|
+
expect = %w(http://0/a http://0/b)
|
629
|
+
assert_equal expect, c.get_paths("f")
|
630
|
+
th.join
|
631
|
+
assert_equal 2, received.size
|
632
|
+
assert_equal received[0], received[1]
|
633
|
+
end
|
634
|
+
|
635
|
+
def test_non_idempotent_command_eof
|
636
|
+
ip = "127.0.0.1"
|
637
|
+
a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
|
638
|
+
hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
|
639
|
+
args = { :hosts => hosts, :domain => "foo" }
|
640
|
+
c = MogileFS::MogileFS.new(args)
|
641
|
+
received = []
|
642
|
+
th = Thread.new do
|
643
|
+
r = IO.select([a, b])
|
644
|
+
x = r[0][0].accept
|
645
|
+
received << x.gets
|
646
|
+
x.close
|
647
|
+
end
|
648
|
+
assert_raises(EOFError) { c.rename("a", "b") }
|
649
|
+
th.join
|
650
|
+
assert_equal 1, received.size
|
651
|
+
end
|
652
|
+
|
653
|
+
def test_list_keys_verbose_ordering # implementation detail
|
654
|
+
received = []
|
655
|
+
sock = TCPServer.new("127.0.0.1", 0)
|
656
|
+
nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
|
657
|
+
c = MogileFS::MogileFS.new(nargs)
|
658
|
+
th = Thread.new do
|
659
|
+
a = sock.accept
|
660
|
+
%w(a b c d e).each do |key|
|
661
|
+
line = a.gets
|
662
|
+
cmd, args = line.split(/\s+/, 2)
|
663
|
+
args = c.backend.url_decode(args.strip)
|
664
|
+
assert_equal "file_info", cmd
|
665
|
+
assert_equal key, args["key"]
|
666
|
+
end
|
667
|
+
out = { "length" => 3, "devcount" => 6 }
|
668
|
+
%w(a b c d e).shuffle.each do |key|
|
669
|
+
out["key"] = key
|
670
|
+
a.write "OK #{c.backend.url_encode(out)}\r\n"
|
671
|
+
end
|
672
|
+
a.close
|
673
|
+
end
|
674
|
+
|
675
|
+
blk = lambda do |key, length, devcount|
|
676
|
+
received << [ key, length, devcount ]
|
677
|
+
end
|
678
|
+
c.list_keys_verbose(%w(a b c d e), blk)
|
679
|
+
th.join
|
680
|
+
received.map! { |(key,_,_)| key }
|
681
|
+
assert_equal %w(a b c d e), received
|
682
|
+
ensure
|
683
|
+
sock.close
|
684
|
+
end
|
685
|
+
|
686
|
+
def test_list_keys_verbose_retry_eof # implementation detail
|
687
|
+
received = []
|
688
|
+
sock = TCPServer.new("127.0.0.1", 0)
|
689
|
+
nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
|
690
|
+
c = MogileFS::MogileFS.new(nargs)
|
691
|
+
th = Thread.new do
|
692
|
+
a = sock.accept
|
693
|
+
%w(a b c d e).each do |key|
|
694
|
+
line = a.gets
|
695
|
+
cmd, args = line.split(/\s+/, 2)
|
696
|
+
args = c.backend.url_decode(args.strip)
|
697
|
+
assert_equal "file_info", cmd
|
698
|
+
assert_equal key, args["key"]
|
699
|
+
end
|
700
|
+
out = { "length" => 3, "devcount" => 6 }
|
701
|
+
%w(d e).each do |key|
|
702
|
+
out["key"] = key
|
703
|
+
a.write "OK #{c.backend.url_encode(out)}\r\n"
|
704
|
+
end
|
705
|
+
a.close # trigger EOF
|
706
|
+
a = sock.accept # client will retry
|
707
|
+
%w(a b c).each do |key|
|
708
|
+
line = a.gets
|
709
|
+
cmd, args = line.split(/\s+/, 2)
|
710
|
+
args = c.backend.url_decode(args.strip)
|
711
|
+
assert_equal "file_info", cmd
|
712
|
+
assert_equal key, args["key"]
|
713
|
+
out["key"] = key
|
714
|
+
a.write "OK #{c.backend.url_encode(out)}\r\n"
|
715
|
+
end
|
716
|
+
a.close
|
717
|
+
end
|
718
|
+
|
719
|
+
blk = lambda do |key, length, devcount|
|
720
|
+
received << [ key, length, devcount ]
|
721
|
+
end
|
722
|
+
c.list_keys_verbose(%w(a b c d e), blk)
|
723
|
+
th.join
|
724
|
+
received.map! { |(key,_,_)| key }
|
725
|
+
assert_equal %w(a b c d e), received
|
726
|
+
ensure
|
727
|
+
sock.close
|
728
|
+
end
|
729
|
+
|
730
|
+
def test_list_keys_verbose_retry_truncated # implementation detail
|
731
|
+
received = []
|
732
|
+
sock = TCPServer.new("127.0.0.1", 0)
|
733
|
+
nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
|
734
|
+
c = MogileFS::MogileFS.new(nargs)
|
735
|
+
th = Thread.new do
|
736
|
+
a = sock.accept
|
737
|
+
%w(a b c d e).each do |key|
|
738
|
+
line = a.gets
|
739
|
+
cmd, args = line.split(/\s+/, 2)
|
740
|
+
args = c.backend.url_decode(args.strip)
|
741
|
+
assert_equal "file_info", cmd
|
742
|
+
assert_equal key, args["key"]
|
743
|
+
end
|
744
|
+
out = { "length" => 3, "devcount" => 6 }
|
745
|
+
out["key"] = "a"
|
746
|
+
a.write "OK #{c.backend.url_encode(out)}\r\n"
|
747
|
+
out["key"] = "b"
|
748
|
+
a.write "OK #{c.backend.url_encode(out)}"
|
749
|
+
a.close # trigger EOF
|
750
|
+
|
751
|
+
a = sock.accept # client will retry
|
752
|
+
%w(b c d e).each do |key|
|
753
|
+
line = a.gets
|
754
|
+
cmd, args = line.split(/\s+/, 2)
|
755
|
+
args = c.backend.url_decode(args.strip)
|
756
|
+
assert_equal "file_info", cmd
|
757
|
+
assert_equal key, args["key"]
|
758
|
+
out["key"] = key
|
759
|
+
a.write "OK #{c.backend.url_encode(out)}\r\n"
|
760
|
+
end
|
761
|
+
a.close
|
762
|
+
end
|
763
|
+
|
764
|
+
blk = lambda do |key, length, devcount|
|
765
|
+
received << [ key, length, devcount ]
|
766
|
+
end
|
767
|
+
c.list_keys_verbose(%w(a b c d e), blk)
|
768
|
+
th.join
|
769
|
+
received.map! { |(key,_,_)| key }
|
770
|
+
assert_equal %w(a b c d e), received
|
771
|
+
ensure
|
772
|
+
sock.close
|
773
|
+
end
|
774
|
+
|
595
775
|
def test_sleep
|
596
776
|
@backend.sleep = {}
|
597
777
|
assert_nothing_raised do
|