mogilefs-client 2.2.0 → 3.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. data/.document +11 -0
  2. data/.gemtest +0 -0
  3. data/.gitignore +4 -0
  4. data/.wrongdoc.yml +5 -0
  5. data/GIT-VERSION-GEN +28 -0
  6. data/GNUmakefile +44 -0
  7. data/HACKING +33 -0
  8. data/{History.txt → History} +0 -1
  9. data/{LICENSE.txt → LICENSE} +0 -1
  10. data/Manifest.txt +34 -7
  11. data/README +51 -0
  12. data/Rakefile +11 -11
  13. data/TODO +10 -0
  14. data/bin/mog +109 -68
  15. data/examples/mogstored_rack.rb +189 -0
  16. data/lib/mogilefs.rb +56 -17
  17. data/lib/mogilefs/admin.rb +128 -62
  18. data/lib/mogilefs/backend.rb +205 -95
  19. data/lib/mogilefs/bigfile.rb +54 -70
  20. data/lib/mogilefs/bigfile/filter.rb +58 -0
  21. data/lib/mogilefs/chunker.rb +30 -0
  22. data/lib/mogilefs/client.rb +0 -2
  23. data/lib/mogilefs/copy_stream.rb +30 -0
  24. data/lib/mogilefs/http_file.rb +175 -0
  25. data/lib/mogilefs/http_reader.rb +79 -0
  26. data/lib/mogilefs/mogilefs.rb +242 -148
  27. data/lib/mogilefs/mysql.rb +3 -4
  28. data/lib/mogilefs/paths_size.rb +24 -0
  29. data/lib/mogilefs/pool.rb +0 -1
  30. data/lib/mogilefs/socket.rb +9 -0
  31. data/lib/mogilefs/socket/kgio.rb +55 -0
  32. data/lib/mogilefs/socket/pure_ruby.rb +70 -0
  33. data/lib/mogilefs/socket_common.rb +58 -0
  34. data/lib/mogilefs/util.rb +6 -169
  35. data/test/aggregate.rb +11 -11
  36. data/test/exec.rb +72 -0
  37. data/test/fresh.rb +222 -0
  38. data/test/integration.rb +43 -0
  39. data/test/setup.rb +1 -0
  40. data/test/socket_test.rb +98 -0
  41. data/test/test_admin.rb +14 -37
  42. data/test/test_backend.rb +50 -107
  43. data/test/test_bigfile.rb +2 -2
  44. data/test/test_db_backend.rb +1 -2
  45. data/test/test_fresh.rb +8 -0
  46. data/test/test_http_reader.rb +34 -0
  47. data/test/test_mogilefs.rb +278 -98
  48. data/test/test_mogilefs_integration.rb +174 -0
  49. data/test/test_mogilefs_integration_large_pipe.rb +62 -0
  50. data/test/test_mogilefs_integration_list_keys.rb +40 -0
  51. data/test/test_mogilefs_socket_kgio.rb +11 -0
  52. data/test/test_mogilefs_socket_pure.rb +7 -0
  53. data/test/test_mogstored_rack.rb +89 -0
  54. data/test/test_mogtool_bigfile.rb +116 -0
  55. data/test/test_mysql.rb +1 -2
  56. data/test/test_pool.rb +1 -1
  57. data/test/test_unit_mogstored_rack.rb +72 -0
  58. metadata +76 -54
  59. data/README.txt +0 -80
  60. data/lib/mogilefs/httpfile.rb +0 -157
  61. data/lib/mogilefs/network.rb +0 -107
  62. data/test/test_network.rb +0 -56
  63. data/test/test_util.rb +0 -121
data/test/test_backend.rb CHANGED
@@ -1,11 +1,9 @@
1
1
  # -*- encoding: binary -*-
2
2
  require 'test/unit'
3
- require 'test/setup'
3
+ require './test/setup'
4
4
 
5
5
  $TESTING = true
6
6
 
7
- require 'mogilefs/backend'
8
-
9
7
  class MogileFS::Backend
10
8
 
11
9
  attr_accessor :hosts
@@ -36,37 +34,18 @@ class TestBackend < Test::Unit::TestCase
36
34
  end
37
35
 
38
36
  def test_do_request
39
- received = Tempfile.new('received')
40
- tmp = TempServer.new(Proc.new do |serv, port|
41
- client, client_addr = serv.accept
42
- client.sync = true
43
- received.syswrite(client.recv(4096))
44
- client.send "OK 1 you=win\r\n", 0
45
- end)
46
-
47
- @backend.hosts = [ "127.0.0.1:#{tmp.port}" ]
48
-
37
+ srv = TCPServer.new("127.0.0.1", 0)
38
+ port = srv.addr[1]
39
+ accepted = Thread.new do
40
+ client = srv.accept
41
+ client.write("OK 1 you=win\r\n")
42
+ client
43
+ end
44
+ @backend.hosts = [ "127.0.0.1:#{port}" ]
49
45
  assert_equal({'you' => 'win'},
50
46
  @backend.do_request('go!', { 'fight' => 'team fight!' }))
51
- received.sysseek(0)
52
- assert_equal "go! fight=team+fight%21\r\n", received.sysread(4096)
53
- ensure
54
- TempServer.destroy_all!
55
- end
56
-
57
- def test_do_request_send_error
58
- socket_request = ''
59
- socket = Object.new
60
- def socket.closed?() false end
61
- def socket.write(request) raise SystemCallError, 'dummy' end
62
-
63
- @backend.instance_variable_set '@socket', socket
64
-
65
- assert_raises MogileFS::UnreachableBackendError do
66
- @backend.do_request 'go!', { 'fight' => 'team fight!' }
67
- end
68
-
69
- assert_equal nil, @backend.instance_variable_get('@socket')
47
+ accepted = accepted.value
48
+ assert_equal "go! fight=team+fight%21\r\n", accepted.readpartial(4096)
70
49
  end
71
50
 
72
51
  def test_automatic_exception
@@ -89,19 +68,6 @@ class TestBackend < Test::Unit::TestCase
89
68
  assert MogileFS::Backend.const_defined?('SizeVerifyError')
90
69
  end
91
70
 
92
- def test_do_request_truncated
93
- socket_request = ''
94
- socket = Object.new
95
- def socket.closed?() false end
96
- def socket.write(request) return request.length - 1 end
97
-
98
- @backend.instance_variable_set '@socket', socket
99
-
100
- assert_raises MogileFS::RequestTruncatedError do
101
- @backend.do_request 'go!', { 'fight' => 'team fight!' }
102
- end
103
- end
104
-
105
71
  def test_make_request
106
72
  assert_equal "go! fight=team+fight%21\r\n",
107
73
  @backend.make_request('go!', { 'fight' => 'team fight!' })
@@ -109,11 +75,11 @@ class TestBackend < Test::Unit::TestCase
109
75
 
110
76
  def test_parse_response
111
77
  assert_equal({'foo' => 'bar', 'baz' => 'hoge'},
112
- @backend.parse_response('OK 1 foo=bar&baz=hoge'))
78
+ @backend.parse_response("OK 1 foo=bar&baz=hoge\r\n"))
113
79
 
114
80
  err = nil
115
81
  begin
116
- @backend.parse_response('ERR you totally suck')
82
+ @backend.parse_response("ERR you totally suck\r\n")
117
83
  rescue MogileFS::Error => err
118
84
  assert_equal 'MogileFS::Backend::YouError', err.class.to_s
119
85
  assert_equal 'totally suck', err.message
@@ -126,6 +92,9 @@ class TestBackend < Test::Unit::TestCase
126
92
  assert_raises MogileFS::InvalidResponseError do
127
93
  @backend.parse_response 'garbage'
128
94
  end
95
+ assert_raises MogileFS::InvalidResponseError do
96
+ @backend.parse_response("OK 1 foo=bar&baz=hoge")
97
+ end
129
98
  end
130
99
 
131
100
  def test_parse_response_newline
@@ -141,89 +110,64 @@ class TestBackend < Test::Unit::TestCase
141
110
  end
142
111
 
143
112
  def test_readable_eh_not_readable
144
- tmp = TempServer.new(Proc.new { |serv,port| serv.accept; sleep })
145
- @backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{tmp.port}" ],
113
+ srv = TCPServer.new("127.0.0.1", 0)
114
+ port = srv.addr[1]
115
+ @backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{port}" ],
146
116
  :timeout => 0.5)
147
117
  begin
148
118
  @backend.do_request 'foo', {}
149
119
  rescue MogileFS::UnreadableSocketError => e
150
- assert_equal "127.0.0.1:#{tmp.port} never became readable", e.message
120
+ assert_equal "127.0.0.1:#{port} never became readable", e.message
151
121
  rescue Exception => err
152
122
  flunk "MogileFS::UnreadableSocketError not raised #{err} #{err.backtrace}"
153
123
  else
154
124
  flunk "MogileFS::UnreadableSocketError not raised"
155
- ensure
156
- TempServer.destroy_all!
157
125
  end
158
126
  end
159
127
 
160
- def test_socket
128
+ def test_socket_dead
161
129
  assert_equal({}, @backend.dead)
162
- assert_raises MogileFS::UnreachableBackendError do @backend.socket end
130
+ assert_raises(MogileFS::UnreachableBackendError) do
131
+ @backend.do_request('test', {})
132
+ end
163
133
  assert_equal(['localhost:1'], @backend.dead.keys)
164
134
  end
165
135
 
166
- def test_socket_robust
167
- bad_accept = Tempfile.new('bad_accept')
168
- accept = Tempfile.new('accept')
169
- bad = Proc.new do |serv,port|
170
- client, client_addr = serv.accept
171
- bad_accept.syswrite('!')
172
- end
173
- good = Proc.new do |serv,port|
174
- client, client_addr = serv.accept
175
- accept.syswrite('.')
176
- client.syswrite('.')
177
- client.close
178
- sleep
179
- end
180
- nr = 10
181
-
182
- nr.times do
183
- begin
184
- t1 = TempServer.new(bad, ENV['TEST_DEAD_PORT'])
185
- t2 = TempServer.new(good)
186
- hosts = ["127.0.0.1:#{t1.port}", "127.0.0.1:#{t2.port}"]
187
- @backend = MogileFS::Backend.new(:hosts => hosts.dup)
188
- assert_equal({}, @backend.dead)
189
- old_chld_handler = trap('CHLD', 'DEFAULT')
190
- t1.destroy!
191
- Process.waitpid(t1.pid)
192
- trap('CHLD', old_chld_handler)
193
- sock = @backend.socket
194
- assert_equal Socket, sock.class
195
- port = Socket.unpack_sockaddr_in(sock.getpeername).first
196
- # p [ 'ports', "port=#{port}", "t1=#{t1.port}", "t2=#{t2.port}" ]
197
- assert_equal t2.port, port
198
- IO.select([sock])
199
- assert_equal '.', sock.sysread(1)
200
- ensure
201
- TempServer.destroy_all!
136
+ def test_socket_robust_on_dead_server
137
+ 10.times do
138
+ t1 = TCPServer.new("127.0.0.1", 0)
139
+ t2 = TCPServer.new("127.0.0.1", 0)
140
+ hosts = ["127.0.0.1:#{t1.addr[1]}", "127.0.0.1:#{t2.addr[1]}"]
141
+ @backend = MogileFS::Backend.new(:hosts => hosts.dup)
142
+ assert_equal({}, @backend.dead)
143
+ t1.close
144
+ thr = Thread.new do
145
+ client = t2.accept
146
+ client.write("OK 1 foo=bar\r\n")
147
+ client
202
148
  end
203
- end # nr.times
204
- assert_equal 0, bad_accept.stat.size
205
- assert_equal nr, accept.stat.size
149
+ rv = nil
150
+ assert_nothing_raised do
151
+ rv = @backend.do_request('test', { "all" => "ALL" })
152
+ end
153
+ accepted = thr.value
154
+ assert_equal "test all=ALL\r\n", accepted.readpartial(666)
155
+ assert_equal({"foo"=>"bar"}, rv)
156
+ end
206
157
  end
207
158
 
208
159
  def test_shutdown
209
- accept_nr = 0
210
- tmp = TempServer.new(Proc.new do |serv,port|
211
- client, client_addr = serv.accept
212
- accept_nr += 1
213
- r = IO.select([client], [client])
214
- client.syswrite(accept_nr.to_s)
215
- sleep
216
- end)
217
- @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
160
+ srv = TCPServer.new('127.0.0.1', 0)
161
+ port = srv.addr[1]
162
+ @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{port}" ]
218
163
  assert @backend.socket
219
164
  assert ! @backend.socket.closed?
220
- IO.select([@backend.socket])
221
- resp = @backend.socket.sysread(4096)
165
+ client = srv.accept
166
+ client.write '1'
167
+ resp = @backend.socket.read(1)
222
168
  @backend.shutdown
223
169
  assert_equal nil, @backend.instance_variable_get(:@socket)
224
170
  assert_equal 1, resp.to_i
225
- ensure
226
- TempServer.destroy_all!
227
171
  end
228
172
 
229
173
  def test_url_decode
@@ -273,7 +217,6 @@ class TestBackend < Test::Unit::TestCase
273
217
  actual = input.map { |c| @backend.url_unescape c }.join "\n"
274
218
 
275
219
  expected = (0..255).map { |c| c.chr }.join "\n"
276
- expected.sub! '+', ' '
277
220
 
278
221
  assert_equal expected, actual
279
222
  end
data/test/test_bigfile.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # -*- encoding: binary -*-
2
- require 'test/setup'
2
+ require './test/setup'
3
3
  require 'mogilefs/bigfile'
4
4
 
5
5
  class TestMogileFS__Bigfile < TestMogileFS
@@ -41,7 +41,7 @@ size 18
41
41
  part 1 bytes=12 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo1:7500/dev2/0/000/144/0000144411.fid, http://foo2:7500/dev1/0/000/144/0000144411.fid
42
42
  part 2 bytes=6 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo4:7500/dev2/0/000/144/0000144411.fid, http://foo3:7500/dev1/0/000/144/0000144411.fid
43
43
  EOS
44
- i = parse_info(s)
44
+ i = bigfile_parse_info(s)
45
45
  assert_equal expect, i
46
46
  end
47
47
 
@@ -1,6 +1,5 @@
1
1
  # -*- encoding: binary -*-
2
- require 'test/setup'
3
- require 'mogilefs/mysql'
2
+ require './test/setup'
4
3
 
5
4
  class TestMogileFS__DbBackend < Test::Unit::TestCase
6
5
  def setup
@@ -0,0 +1,8 @@
1
+ # -*- encoding: binary -*-
2
+ require "./test/fresh"
3
+
4
+ class TestMogFresh < Test::Unit::TestCase
5
+ include TestFreshSetup
6
+ alias setup setup_mogilefs
7
+ alias teardown teardown_mogilefs
8
+ end
@@ -0,0 +1,34 @@
1
+ # -*- encoding: binary -*-
2
+ require "./test/setup"
3
+
4
+ class TestHTTPReader < Test::Unit::TestCase
5
+ def rsock
6
+ host = "127.0.0.1"
7
+ s = TCPServer.new(host, 0)
8
+ th = Thread.new do
9
+ c = s.accept
10
+ c.readpartial(0x666)
11
+ c.write("HTTP/1.0 200 OK\r\nContent-Length: 666\r\n\r\n666")
12
+ c.close
13
+ end
14
+ path = "http://#{host}:#{s.addr[1]}/"
15
+ r = MogileFS::HTTPReader.try(path, 666, nil)
16
+ assert_nil th.value
17
+ assert_kind_of IO, r
18
+ r
19
+ ensure
20
+ s.close
21
+ end
22
+
23
+ def test_short_to_s
24
+ r = rsock
25
+ assert_raises(MogileFS::SizeMismatchError) { r.to_s }
26
+ r.close
27
+ end
28
+
29
+ def test_short_stream_to
30
+ r = rsock
31
+ assert_raises(MogileFS::SizeMismatchError) { r.stream_to("/dev/null") }
32
+ r.close
33
+ end
34
+ end
@@ -1,12 +1,10 @@
1
1
  # -*- encoding: binary -*-
2
- require 'test/setup'
2
+ require './test/setup'
3
3
  require 'stringio'
4
4
  require 'tempfile'
5
5
  require 'fileutils'
6
6
 
7
7
  class TestMogileFS__MogileFS < TestMogileFS
8
- include MogileFS::Util
9
-
10
8
  def setup
11
9
  @klass = MogileFS::MogileFS
12
10
  super
@@ -24,7 +22,7 @@ class TestMogileFS__MogileFS < TestMogileFS
24
22
  tmp = Tempfile.new('accept')
25
23
  accept = File.open(tmp.path, "ab")
26
24
  svr = Proc.new do |serv, port|
27
- client, client_addr = serv.accept
25
+ client, _ = serv.accept
28
26
  client.sync = true
29
27
  readed = client.recv(4096, 0)
30
28
  assert(readed =~ \
@@ -50,7 +48,7 @@ class TestMogileFS__MogileFS < TestMogileFS
50
48
  tmp = Tempfile.new('accept')
51
49
  accept = File.open(tmp.path, 'ab')
52
50
  svr1 = Proc.new do |serv, port|
53
- client, client_addr = serv.accept
51
+ client, _ = serv.accept
54
52
  client.sync = true
55
53
  readed = client.recv(4096, 0)
56
54
  assert(readed =~ \
@@ -61,7 +59,7 @@ class TestMogileFS__MogileFS < TestMogileFS
61
59
  end
62
60
 
63
61
  svr2 = Proc.new do |serv, port|
64
- client, client_addr = serv.accept
62
+ client, _ = serv.accept
65
63
  client.sync = true
66
64
  readed = client.recv(4096, 0)
67
65
  assert(readed =~ \
@@ -97,13 +95,13 @@ class TestMogileFS__MogileFS < TestMogileFS
97
95
 
98
96
  accept = Tempfile.new('accept')
99
97
  svr = Proc.new do |serv, port|
100
- client, client_addr = serv.accept
98
+ client, _ = serv.accept
101
99
  client.sync = true
102
100
  accept.syswrite('.')
103
101
  readed = client.recv(4096, 0)
104
102
  assert(readed =~ \
105
103
  %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
106
- sysrwloop(tmpfp, client)
104
+ MogileFS.io.copy_stream(tmpfp, client)
107
105
  client.close
108
106
  exit 0
109
107
  end
@@ -210,40 +208,6 @@ class TestMogileFS__MogileFS < TestMogileFS
210
208
  assert_equal 'new_key_2', next_after
211
209
  end
212
210
 
213
- def test_list_keys_block
214
- @backend.list_keys = { 'key_count' => '2', 'next_after' => 'new_key_2',
215
- 'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
216
- http_resp = "HTTP/1.0 200 OK\r\nContent-Length: %u\r\n"
217
- srv = Proc.new do |serv, port, size|
218
- client, client_addr = serv.accept
219
- client.sync = true
220
- readed = client.readpartial(4096)
221
- assert %r{\AHEAD } =~ readed
222
- client.send(http_resp % size, 0)
223
- client.close
224
- end
225
- t1 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
226
- t2 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
227
- t3 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 10) })
228
- @backend.get_paths = { 'paths' => '2',
229
- 'path1' => "http://127.0.0.1:#{t1.port}/",
230
- 'path2' => "http://127.0.0.1:#{t2.port}/" }
231
- @backend.get_paths = { 'paths' => '1',
232
- 'path1' => "http://127.0.0.1:#{t3.port}/" }
233
-
234
- res = []
235
- keys, next_after = @client.list_keys('new') do |key,length,devcount|
236
- res << [ key, length, devcount ]
237
- end
238
-
239
- expect_res = [ [ 'new_key_1', 5, 2 ], [ 'new_key_2', 10, 1 ] ]
240
- assert_equal expect_res, res
241
- assert_equal ['new_key_1', 'new_key_2'], keys.sort
242
- assert_equal 'new_key_2', next_after
243
- ensure
244
- TempServer.destroy_all!
245
- end
246
-
247
211
  def test_new_file_http
248
212
  @client.readonly = true
249
213
  assert_raises MogileFS::ReadOnlyError do
@@ -258,44 +222,6 @@ class TestMogileFS__MogileFS < TestMogileFS
258
222
  end
259
223
  end
260
224
 
261
- def test_size_http
262
- accept = Tempfile.new('accept')
263
- t = TempServer.new(Proc.new do |serv,port|
264
- client, client_addr = serv.accept
265
- client.sync = true
266
- readed = client.recv(4096, 0) rescue nil
267
- accept.syswrite('.')
268
- assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
269
- client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\n", 0)
270
- client.close
271
- end)
272
-
273
- path = "http://127.0.0.1:#{t.port}/path"
274
- @backend.get_paths = { 'paths' => 1, 'path1' => path }
275
-
276
- assert_equal 5, @client.size('key')
277
- assert_equal 1, accept.stat.size
278
- end
279
-
280
- def test_bad_size_http
281
- tmp = Tempfile.new('accept')
282
- t = TempServer.new(Proc.new do |serv,port|
283
- client, client_addr = serv.accept
284
- client.sync = true
285
- readed = client.recv(4096, 0) rescue nil
286
- assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
287
- tmp.syswrite('.')
288
- client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
289
- client.close
290
- end)
291
-
292
- path = "http://127.0.0.1:#{t.port}/path"
293
- @backend.get_paths = { 'paths' => 1, 'path1' => path }
294
-
295
- assert_nil @client.size('key')
296
- assert_equal 1, tmp.stat.size
297
- end
298
-
299
225
  def test_store_file_small_http
300
226
  received = Tempfile.new('received')
301
227
  to_store = Tempfile.new('small')
@@ -303,9 +229,9 @@ class TestMogileFS__MogileFS < TestMogileFS
303
229
 
304
230
  expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
305
231
  t = TempServer.new(Proc.new do |serv, accept|
306
- client, client_addr = serv.accept
232
+ client, _ = serv.accept
307
233
  client.sync = true
308
- received.syswrite(client.recv(4096, 0))
234
+ received.syswrite(client.read(expected.bytesize))
309
235
  client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
310
236
  client.close
311
237
  end)
@@ -327,9 +253,14 @@ class TestMogileFS__MogileFS < TestMogileFS
327
253
  expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
328
254
 
329
255
  t = TempServer.new(Proc.new do |serv, accept|
330
- client, client_addr = serv.accept
256
+ client, _ = serv.accept
331
257
  client.sync = true
332
- received.syswrite(client.recv(4096, 0))
258
+ seen = ""
259
+ while seen !~ /\r\n\r\ndata/
260
+ buf = client.readpartial(4096)
261
+ seen << buf
262
+ received.syswrite(buf)
263
+ end
333
264
  client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
334
265
  client.close
335
266
  end)
@@ -357,7 +288,7 @@ class TestMogileFS__MogileFS < TestMogileFS
357
288
  expected += "data"
358
289
  end
359
290
  t = TempServer.new(Proc.new do |serv, accept|
360
- client, client_addr = serv.accept
291
+ client, _ = serv.accept
361
292
  client.sync = true
362
293
  nr = 0
363
294
  loop do
@@ -391,23 +322,35 @@ class TestMogileFS__MogileFS < TestMogileFS
391
322
  TempServer.destroy_all!
392
323
  end
393
324
 
394
- def test_store_content_multi_dest_failover
325
+ def test_store_content_multi_dest_failover_path
326
+ test_store_content_multi_dest_failover(true)
327
+ end
328
+
329
+ def test_store_content_multi_dest_failover(big_io = false)
395
330
  received1 = Tempfile.new('received')
396
331
  received2 = Tempfile.new('received')
397
332
  expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
398
333
 
399
334
  t1 = TempServer.new(Proc.new do |serv, accept|
400
- client, client_addr = serv.accept
401
- client.sync = true
402
- received1.syswrite(client.recv(4096, 0))
335
+ client, _ = serv.accept
336
+ seen = ""
337
+ while seen !~ /\r\n\r\ndata/
338
+ buf = client.readpartial(4096)
339
+ seen << buf
340
+ received1.syswrite(buf)
341
+ end
403
342
  client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
404
343
  client.close
405
344
  end)
406
345
 
407
346
  t2 = TempServer.new(Proc.new do |serv, accept|
408
- client, client_addr = serv.accept
409
- client.sync = true
410
- received2.syswrite(client.recv(4096, 0))
347
+ client, _ = serv.accept
348
+ seen = ""
349
+ while seen !~ /\r\n\r\ndata/
350
+ buf = client.readpartial(4096)
351
+ seen << buf
352
+ received2.syswrite(buf)
353
+ end
411
354
  client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
412
355
  client.close
413
356
  end)
@@ -420,7 +363,15 @@ class TestMogileFS__MogileFS < TestMogileFS
420
363
  'path_2' => "http://127.0.0.1:#{t2.port}/path",
421
364
  }
422
365
 
423
- nr = @client.store_content 'new_key', 'test', 'data'
366
+ if big_io
367
+ tmp = Tempfile.new('data')
368
+ tmp.sync = true
369
+ tmp.write 'data'
370
+ nr = @client.store_file('new_key', 'test', tmp.path)
371
+ tmp.close!
372
+ else
373
+ nr = @client.store_content 'new_key', 'test', 'data'
374
+ end
424
375
  assert_equal 4, nr
425
376
  received1.sysseek(0)
426
377
  received2.sysseek(0)
@@ -432,7 +383,7 @@ class TestMogileFS__MogileFS < TestMogileFS
432
383
 
433
384
  def test_store_content_http_fail
434
385
  t = TempServer.new(Proc.new do |serv, accept|
435
- client, client_addr = serv.accept
386
+ client, _ = serv.accept
436
387
  client.sync = true
437
388
  client.recv(4096, 0)
438
389
  client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
@@ -444,7 +395,7 @@ class TestMogileFS__MogileFS < TestMogileFS
444
395
  'path' => "http://127.0.0.1:#{t.port}/path",
445
396
  }
446
397
 
447
- assert_raises MogileFS::HTTPFile::BadResponseError do
398
+ assert_raises MogileFS::HTTPFile::NoStorageNodesError do
448
399
  @client.store_content 'new_key', 'test', 'data'
449
400
  end
450
401
  end
@@ -453,7 +404,7 @@ class TestMogileFS__MogileFS < TestMogileFS
453
404
  received = Tempfile.new('received')
454
405
  expected = "PUT /path HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
455
406
  t = TempServer.new(Proc.new do |serv, accept|
456
- client, client_addr = serv.accept
407
+ client, _ = serv.accept
457
408
  client.sync = true
458
409
  received.syswrite(client.recv(4096, 0))
459
410
  client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
@@ -503,7 +454,7 @@ class TestMogileFS__MogileFS < TestMogileFS
503
454
 
504
455
  readed = Tempfile.new('readed')
505
456
  t = TempServer.new(Proc.new do |serv, accept|
506
- client, client_addr = serv.accept
457
+ client, _ = serv.accept
507
458
  client.sync = true
508
459
  nr = 0
509
460
  loop do
@@ -525,7 +476,7 @@ class TestMogileFS__MogileFS < TestMogileFS
525
476
 
526
477
  orig_size = to_put.size
527
478
  nr = @client.store_file('new_key', 'test', to_put.path)
528
- assert nr
479
+ assert nr, nr.inspect
529
480
  assert_equal orig_size, nr
530
481
  assert_equal orig_size, to_put.size
531
482
  readed.sysseek(0)
@@ -592,6 +543,235 @@ class TestMogileFS__MogileFS < TestMogileFS
592
543
  assert_equal 'readonly mogilefs', e.message
593
544
  end
594
545
 
546
+ def assert_get_paths_args(expect, *args)
547
+ sock = TCPServer.new("127.0.0.1", 0)
548
+ nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
549
+ c = MogileFS::MogileFS.new(nargs)
550
+ received = []
551
+ th = Thread.new do
552
+ a = sock.accept
553
+ line = a.gets
554
+ received << line
555
+ a.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
556
+ a.close
557
+ end
558
+ paths_expect = %w(http://0/a http://0/b)
559
+ assert_equal paths_expect, c.get_paths("f", *args)
560
+ th.join
561
+ assert_equal 1, received.size
562
+ tmp = c.backend.url_decode(received[0].split(/\s+/)[1])
563
+ assert_equal "f", tmp.delete("key")
564
+ assert_equal "foo", tmp.delete("domain")
565
+ assert_equal expect, tmp
566
+ c.backend.shutdown
567
+ ensure
568
+ sock.close
569
+ end
570
+
571
+ def test_get_paths_args
572
+ assert_get_paths_args({"noverify"=>"1", "zone"=>""})
573
+ assert_get_paths_args({"noverify"=>"0", "zone"=>""}, false)
574
+ assert_get_paths_args({"noverify"=>"0", "zone"=>""}, :noverify=>false)
575
+ assert_get_paths_args({"noverify"=>"1", "zone"=>"alt"}, true, "alt")
576
+ assert_get_paths_args({"noverify"=>"1", "zone"=>"alt"},
577
+ {:noverify => true, :zone => "alt"})
578
+ assert_get_paths_args({"noverify"=>"1", "zone"=>"alt","pathcount"=>"666"},
579
+ {:noverify => true, :zone => "alt", :pathcount=>666})
580
+ end
581
+
582
+ def test_idempotent_command_eof
583
+ ip = "127.0.0.1"
584
+ a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
585
+ hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
586
+ args = { :hosts => hosts, :domain => "foo" }
587
+ c = MogileFS::MogileFS.new(args)
588
+ received = []
589
+ th = Thread.new do
590
+ r = IO.select([a, b])
591
+ x = r[0][0].accept
592
+ received << x.gets
593
+ x.close
594
+
595
+ r = IO.select([a, b])
596
+ x = r[0][0].accept
597
+ received << x.gets
598
+ x.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
599
+ x.close
600
+ end
601
+ expect = %w(http://0/a http://0/b)
602
+ assert_equal expect, c.get_paths("f")
603
+ th.join
604
+ assert_equal 2, received.size
605
+ assert_equal received[0], received[1]
606
+ end
607
+
608
+ def test_idempotent_command_response_truncated
609
+ ip = "127.0.0.1"
610
+ a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
611
+ hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
612
+ args = { :hosts => hosts, :domain => "foo" }
613
+ c = MogileFS::MogileFS.new(args)
614
+ received = []
615
+ th = Thread.new do
616
+ r = IO.select([a, b])
617
+ x = r[0][0].accept
618
+ received << x.gets
619
+ x.write("OK paths=2&path1=http://0/a&path2=http://0/")
620
+ x.close
621
+
622
+ r = IO.select([a, b])
623
+ x = r[0][0].accept
624
+ received << x.gets
625
+ x.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
626
+ x.close
627
+ end
628
+ expect = %w(http://0/a http://0/b)
629
+ assert_equal expect, c.get_paths("f")
630
+ th.join
631
+ assert_equal 2, received.size
632
+ assert_equal received[0], received[1]
633
+ end
634
+
635
+ def test_non_idempotent_command_eof
636
+ ip = "127.0.0.1"
637
+ a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
638
+ hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
639
+ args = { :hosts => hosts, :domain => "foo" }
640
+ c = MogileFS::MogileFS.new(args)
641
+ received = []
642
+ th = Thread.new do
643
+ r = IO.select([a, b])
644
+ x = r[0][0].accept
645
+ received << x.gets
646
+ x.close
647
+ end
648
+ assert_raises(EOFError) { c.rename("a", "b") }
649
+ th.join
650
+ assert_equal 1, received.size
651
+ end
652
+
653
+ def test_list_keys_verbose_ordering # implementation detail
654
+ received = []
655
+ sock = TCPServer.new("127.0.0.1", 0)
656
+ nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
657
+ c = MogileFS::MogileFS.new(nargs)
658
+ th = Thread.new do
659
+ a = sock.accept
660
+ %w(a b c d e).each do |key|
661
+ line = a.gets
662
+ cmd, args = line.split(/\s+/, 2)
663
+ args = c.backend.url_decode(args.strip)
664
+ assert_equal "file_info", cmd
665
+ assert_equal key, args["key"]
666
+ end
667
+ out = { "length" => 3, "devcount" => 6 }
668
+ %w(a b c d e).shuffle.each do |key|
669
+ out["key"] = key
670
+ a.write "OK #{c.backend.url_encode(out)}\r\n"
671
+ end
672
+ a.close
673
+ end
674
+
675
+ blk = lambda do |key, length, devcount|
676
+ received << [ key, length, devcount ]
677
+ end
678
+ c.list_keys_verbose(%w(a b c d e), blk)
679
+ th.join
680
+ received.map! { |(key,_,_)| key }
681
+ assert_equal %w(a b c d e), received
682
+ ensure
683
+ sock.close
684
+ end
685
+
686
+ def test_list_keys_verbose_retry_eof # implementation detail
687
+ received = []
688
+ sock = TCPServer.new("127.0.0.1", 0)
689
+ nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
690
+ c = MogileFS::MogileFS.new(nargs)
691
+ th = Thread.new do
692
+ a = sock.accept
693
+ %w(a b c d e).each do |key|
694
+ line = a.gets
695
+ cmd, args = line.split(/\s+/, 2)
696
+ args = c.backend.url_decode(args.strip)
697
+ assert_equal "file_info", cmd
698
+ assert_equal key, args["key"]
699
+ end
700
+ out = { "length" => 3, "devcount" => 6 }
701
+ %w(d e).each do |key|
702
+ out["key"] = key
703
+ a.write "OK #{c.backend.url_encode(out)}\r\n"
704
+ end
705
+ a.close # trigger EOF
706
+ a = sock.accept # client will retry
707
+ %w(a b c).each do |key|
708
+ line = a.gets
709
+ cmd, args = line.split(/\s+/, 2)
710
+ args = c.backend.url_decode(args.strip)
711
+ assert_equal "file_info", cmd
712
+ assert_equal key, args["key"]
713
+ out["key"] = key
714
+ a.write "OK #{c.backend.url_encode(out)}\r\n"
715
+ end
716
+ a.close
717
+ end
718
+
719
+ blk = lambda do |key, length, devcount|
720
+ received << [ key, length, devcount ]
721
+ end
722
+ c.list_keys_verbose(%w(a b c d e), blk)
723
+ th.join
724
+ received.map! { |(key,_,_)| key }
725
+ assert_equal %w(a b c d e), received
726
+ ensure
727
+ sock.close
728
+ end
729
+
730
+ def test_list_keys_verbose_retry_truncated # implementation detail
731
+ received = []
732
+ sock = TCPServer.new("127.0.0.1", 0)
733
+ nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
734
+ c = MogileFS::MogileFS.new(nargs)
735
+ th = Thread.new do
736
+ a = sock.accept
737
+ %w(a b c d e).each do |key|
738
+ line = a.gets
739
+ cmd, args = line.split(/\s+/, 2)
740
+ args = c.backend.url_decode(args.strip)
741
+ assert_equal "file_info", cmd
742
+ assert_equal key, args["key"]
743
+ end
744
+ out = { "length" => 3, "devcount" => 6 }
745
+ out["key"] = "a"
746
+ a.write "OK #{c.backend.url_encode(out)}\r\n"
747
+ out["key"] = "b"
748
+ a.write "OK #{c.backend.url_encode(out)}"
749
+ a.close # trigger EOF
750
+
751
+ a = sock.accept # client will retry
752
+ %w(b c d e).each do |key|
753
+ line = a.gets
754
+ cmd, args = line.split(/\s+/, 2)
755
+ args = c.backend.url_decode(args.strip)
756
+ assert_equal "file_info", cmd
757
+ assert_equal key, args["key"]
758
+ out["key"] = key
759
+ a.write "OK #{c.backend.url_encode(out)}\r\n"
760
+ end
761
+ a.close
762
+ end
763
+
764
+ blk = lambda do |key, length, devcount|
765
+ received << [ key, length, devcount ]
766
+ end
767
+ c.list_keys_verbose(%w(a b c d e), blk)
768
+ th.join
769
+ received.map! { |(key,_,_)| key }
770
+ assert_equal %w(a b c d e), received
771
+ ensure
772
+ sock.close
773
+ end
774
+
595
775
  def test_sleep
596
776
  @backend.sleep = {}
597
777
  assert_nothing_raised do