mogilefs-client 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,7 +24,7 @@ class MogileFS::Socket < Socket
24
24
 
25
25
  def timed_read(len, dst = "", timeout = 5)
26
26
  begin
27
- IO.select([self], nil, nil, timeout) or unreadable_socket!
27
+ IO.select([self], nil, nil, timeout) or unreadable_socket!(timeout)
28
28
  return read_nonblock(len, dst)
29
29
  rescue Errno::EAGAIN
30
30
  rescue EOFError
@@ -34,7 +34,7 @@ class MogileFS::Socket < Socket
34
34
 
35
35
  def timed_peek(len, dst, timeout = 5)
36
36
  begin
37
- IO.select([self], nil, nil, timeout) or unreadable_socket!
37
+ IO.select([self], nil, nil, timeout) or unreadable_socket!(timeout)
38
38
  rc = recv_nonblock(len, Socket::MSG_PEEK)
39
39
  return rc.empty? ? nil : dst.replace(rc)
40
40
  rescue Errno::EAGAIN
@@ -62,7 +62,7 @@ class MogileFS::Socket < Socket
62
62
  end
63
63
  rescue Errno::EAGAIN
64
64
  IO.select(nil, [self], nil, timeout) or
65
- request_truncated!(written, expect)
65
+ request_truncated!(written, expect, timeout)
66
66
  end while true
67
67
  end
68
68
  end
@@ -12,14 +12,15 @@ module MogileFS::SocketCommon
12
12
  self
13
13
  end
14
14
 
15
- def unreadable_socket!
15
+ def unreadable_socket!(timeout)
16
16
  raise MogileFS::UnreadableSocketError,
17
- "#@mogilefs_addr never became readable"
17
+ "#@mogilefs_addr never became readable (timeout=#{timeout.inspect})"
18
18
  end
19
19
 
20
- def request_truncated!(written, expect)
20
+ def request_truncated!(written, expect, timeout)
21
+ timeout = timeout.inspect
21
22
  raise MogileFS::RequestTruncatedError,
22
- "request truncated (sent #{written} expected #{expect})"
23
+ "request truncated (sent #{written} expected #{expect} timeout=#{timeout})"
23
24
  end
24
25
 
25
26
  SEP_RE = /\A(.*?#{Regexp.escape("\n")})/
@@ -43,9 +44,9 @@ module MogileFS::SocketCommon
43
44
  def read(size, buf = "", timeout = 5)
44
45
  timed_read(size, buf, timeout) or return # nil/EOF
45
46
 
46
- while size > buf.bytesize
47
+ while (size -= buf.bytesize) > 0
47
48
  tmp ||= ""
48
- timed_read(size - buf.bytesize, tmp, timeout) or return buf # truncated
49
+ timed_read(size, tmp, timeout) or return buf # truncated
49
50
  buf << tmp
50
51
  end
51
52
 
data/test/fresh.rb CHANGED
@@ -120,8 +120,8 @@ EOF
120
120
  args = { :ip => @test_host, :port => @mogstored_http_port }
121
121
  args[:status] = "alive"
122
122
  @admin.create_host("me", args)
123
- Dir.mkdir("#@docroot/dev1")
124
- Dir.mkdir("#@docroot/dev2")
123
+ assert File.directory?("#@docroot/dev1")
124
+ assert File.directory?("#@docroot/dev2")
125
125
  yield_for_monitor_update { @admin.get_hosts.empty? or break }
126
126
 
127
127
  # TODO: allow adding devices via our MogileFS::Admin class
@@ -136,6 +136,7 @@ EOF
136
136
  out.close! if out
137
137
  err.close! if err
138
138
  status, out, err = mogadm("check")
139
+ assert status.success?, status.inspect
139
140
  if (tries += 1) > 100
140
141
  warn err.read
141
142
  puts out.read
@@ -198,6 +199,8 @@ EOF
198
199
 
199
200
  def setup_mogstored
200
201
  @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
202
+ Dir.mkdir("#@docroot/dev1")
203
+ Dir.mkdir("#@docroot/dev2")
201
204
  @mogstored_mgmt = TCPServer.new(@test_host, 0)
202
205
  @mogstored_http = TCPServer.new(@test_host, 0)
203
206
  @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
data/test/test_backend.rb CHANGED
@@ -58,14 +58,14 @@ class TestBackend < Test::Unit::TestCase
58
58
  assert @backend.error('peb_kac')
59
59
  assert_equal MogileFS::Error, @backend.error('PebKacError').superclass
60
60
  assert MogileFS::Backend.const_defined?('PebKacError')
61
- end
62
61
 
63
- def test_size_verify_error_defined
64
- # "ErrorError" just looks dumb, but we used to get it
65
- # since mogilefs would send us "size_verify_error" and we'd
66
- # blindly append "Error" to the exception
67
- assert ! MogileFS::Backend.const_defined?('SizeVerifyErrorError')
68
- assert MogileFS::Backend.const_defined?('SizeVerifyError')
62
+ assert_nothing_raised do
63
+ MogileFS::Backend::OMFGWTFBBQError
64
+ end
65
+ assert_equal MogileFS::Error, MogileFS::Backend::OMFGWTFBBQError.superclass
66
+ assert_raises(NameError) do
67
+ MogileFS::Backend::FailFailFail
68
+ end
69
69
  end
70
70
 
71
71
  def test_make_request
@@ -117,7 +117,7 @@ class TestBackend < Test::Unit::TestCase
117
117
  begin
118
118
  @backend.do_request 'foo', {}
119
119
  rescue MogileFS::UnreadableSocketError => e
120
- assert_equal "127.0.0.1:#{port} never became readable", e.message
120
+ assert_match(/127\.0\.0\.1:#{port} never became readable/, e.message)
121
121
  rescue Exception => err
122
122
  flunk "MogileFS::UnreadableSocketError not raised #{err} #{err.backtrace}"
123
123
  else
@@ -221,5 +221,13 @@ class TestBackend < Test::Unit::TestCase
221
221
  assert_equal expected, actual
222
222
  end
223
223
 
224
+ def test_fail_timeout
225
+ o = { :domain => "none", :hosts => %w(0:666 0:6 0:66) }
226
+ c = MogileFS::MogileFS.new(o)
227
+ assert_equal 5, c.backend.instance_variable_get(:@fail_timeout)
228
+ o[:fail_timeout] = 0.666
229
+ c = MogileFS::MogileFS.new(o)
230
+ assert_equal 0.666, c.backend.instance_variable_get(:@fail_timeout)
231
+ end
224
232
  end
225
233
 
@@ -8,7 +8,9 @@ class TestMogileFSIntegration < TestMogIntegration
8
8
  end
9
9
 
10
10
  def test_CRUD
11
+ assert ! @client.exist?("CRUD")
11
12
  assert_equal 4, @client.store_content("CRUD", "default", "DATA")
13
+ assert @client.exist?("CRUD")
12
14
  assert_equal 4, @client.size("CRUD")
13
15
  assert_equal "DATA", @client.get_file_data("CRUD")
14
16
  assert_equal "DAT", @client.get_file_data("CRUD", nil, 3)
@@ -65,6 +67,9 @@ class TestMogileFSIntegration < TestMogIntegration
65
67
  def tmp.size
66
68
  666
67
69
  end
70
+ def tmp.read(len, buf = "")
71
+ raise Errno::EIO
72
+ end
68
73
 
69
74
  assert_raises(MogileFS::HTTPFile::NonRetryableError) do
70
75
  @client.store_file("non_rewindable", nil, tmp)
@@ -169,4 +174,120 @@ class TestMogileFSIntegration < TestMogIntegration
169
174
  end
170
175
  assert_equal count, seen.size
171
176
  end if ENV["TEST_EXPENSIVE"]
177
+
178
+ def test_new_file_no_block
179
+ rv = @client.new_file("no_block")
180
+ assert_nothing_raised { rv.write "HELLO" }
181
+ assert_nil rv.close
182
+ assert_equal "HELLO", @client.get_file_data("no_block")
183
+ end
184
+
185
+ def test_new_file_known_content_length
186
+ rv = @client.new_file("a", :content_length => 5)
187
+ assert_nothing_raised { rv.write "HELLO" }
188
+ assert_nil rv.close
189
+ assert_equal "HELLO", @client.get_file_data("a")
190
+
191
+ rv = @client.new_file("a", :content_length => 6)
192
+ assert_nothing_raised { rv.write "GOOD" }
193
+ assert_raises(MogileFS::SizeMismatchError) { rv.close }
194
+ assert_equal "HELLO", @client.get_file_data("a")
195
+
196
+ rv = @client.new_file("large", :content_length => 6, :largefile => true)
197
+ assert_instance_of MogileFS::NewFile::Stream, rv
198
+ assert_equal 6, rv.write("HIHIHI")
199
+ assert_nil rv.close
200
+ assert_equal "HIHIHI", @client.get_file_data("large")
201
+ end
202
+
203
+ def test_new_file_content_md5
204
+ r, w = IO.pipe
205
+ b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
206
+ rv = @client.new_file("a", :content_md5 => b64digest, :content_length => 5)
207
+ assert_nothing_raised { rv.write "HELLO" }
208
+ assert_nil rv.close
209
+ assert_equal "HELLO", @client.get_file_data("a")
210
+
211
+ assert_nothing_raised { w.write "HIHI"; w.close }
212
+ assert_raises(ArgumentError) do
213
+ @client.new_file("a", :content_md5 => b64digest) { |f| f.big_io = r }
214
+ end
215
+ assert_equal "HELLO", @client.get_file_data("a")
216
+
217
+ assert_nothing_raised do
218
+ @client.new_file("a", :content_md5 => :trailer) { |f| f.big_io = r }
219
+ end
220
+ assert_equal "HIHI", @client.get_file_data("a")
221
+
222
+ # legacy, in case anybody used it
223
+ rv = @client.new_file("a",{:class => "default"}, 6)
224
+ assert_equal 2, rv.write("HI")
225
+ assert_raises(MogileFS::SizeMismatchError) { rv.close }
226
+ assert_equal "HIHI", @client.get_file_data("a")
227
+
228
+ rv = @client.new_file("a",{:class => "default"}, 2)
229
+ assert_equal 2, rv.write("HI")
230
+ assert_nil rv.close
231
+ assert_equal "HI", @client.get_file_data("a")
232
+ assert_raises(MogileFS::Backend::UnregClassError) {
233
+ @client.new_file("a", "non-existent", 2)
234
+ }
235
+ assert_raises(MogileFS::Backend::UnregClassError) {
236
+ @client.new_file("a", :class => "non-existent")
237
+ }
238
+ ensure
239
+ r.close if r
240
+ end
241
+
242
+ def test_store_content_opts
243
+ b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
244
+ assert_nothing_raised do
245
+ @client.store_content("c", nil, "HELLO", :content_md5 => b64digest)
246
+ end
247
+ assert_raises(MogileFS::SizeMismatchError) do
248
+ @client.store_content("c", nil, "GOODBYE", :content_length => 2)
249
+ end
250
+ assert_equal "HELLO", @client.get_file_data("c")
251
+ end
252
+
253
+ def test_store_file_opts
254
+ b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
255
+ io = StringIO.new("HELLO")
256
+ assert_nothing_raised do
257
+ @client.store_file("c", nil, io, :content_md5 => b64digest)
258
+ end
259
+
260
+ io = StringIO.new("GOODBYE")
261
+ assert_raises(MogileFS::SizeMismatchError) do
262
+ @client.store_content("c", nil, io, :content_length => 2)
263
+ end
264
+ assert_equal "HELLO", @client.get_file_data("c")
265
+ end
266
+
267
+ def test_store_file_content_md5_lambda
268
+ checked = false
269
+ expect_md5 = lambda do
270
+ checked = true
271
+ [ Digest::MD5.digest("HELLO") ].pack('m').strip
272
+ end
273
+ io = StringIO.new("HELLO")
274
+ assert_nothing_raised do
275
+ @client.store_file("c", nil, io, :content_md5 => expect_md5)
276
+ end
277
+
278
+ assert_equal true, checked, "expect_md5 lambda called"
279
+ end
280
+
281
+ def test_store_file_unlinked_tempfile
282
+ tmp = Tempfile.new("store_file_unlinked")
283
+ tmp.unlink
284
+ tmp.sync = true
285
+ tmp.write "HIHI"
286
+ tmp.rewind
287
+ assert_nothing_raised do
288
+ @client.store_file("unlinked", nil, tmp)
289
+ end
290
+
291
+ assert_equal "HIHI", @client.get_file_data("unlinked")
292
+ end
172
293
  end
@@ -1,5 +1,12 @@
1
1
  # -*- encoding: binary -*-
2
2
  require "./test/fresh"
3
+ begin
4
+ require 'mogstored_rack'
5
+ require 'unicorn'
6
+ ok = true
7
+ rescue LoadError
8
+ ok = false
9
+ end
3
10
 
4
11
  class TestMogstoredRack < Test::Unit::TestCase
5
12
  include TestFreshSetup
@@ -7,6 +14,106 @@ class TestMogstoredRack < Test::Unit::TestCase
7
14
  setup_mogilefs
8
15
  end
9
16
 
17
+ def test_range_put_new_file
18
+ add_host_device_domain
19
+ client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
20
+
21
+ io = client.new_file "range0", :largefile => :content_range
22
+ assert_nil io.close
23
+ assert_equal "", client.get_file_data("range0")
24
+
25
+ io = client.new_file "writes", :largefile => :content_range
26
+ %w(a b c d e).each { |x| io.write(x) }
27
+ assert_nil io.close
28
+ assert_equal "abcde", client.get_file_data("writes")
29
+
30
+ io = client.new_file "puts", :largefile => :content_range
31
+ %w(a b c d e).each { |x| io.puts(x) }
32
+ assert ! client.exist?("puts")
33
+ assert_nil io.close
34
+ assert_equal "a\nb\nc\nd\ne\n", client.get_file_data("puts")
35
+ end
36
+
37
+ def test_stream_new_file
38
+ add_host_device_domain
39
+ client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
40
+ client.new_file("chunky", :largefile => :stream) do |io|
41
+ assert_instance_of MogileFS::NewFile::Stream, io
42
+ assert_equal(5, io.write("HELLO"))
43
+ assert_nil io.md5
44
+ end
45
+ assert_equal "HELLO", client.get_file_data("chunky")
46
+
47
+ io = client.new_file("puts", :largefile => :stream)
48
+ assert_instance_of MogileFS::NewFile::Stream, io
49
+ assert_equal io, IO.select(nil, [io])[1][0], "IO.select-able"
50
+
51
+ assert_nil(io.puts("PUTS!"))
52
+ assert_nil(io.puts("PUTZ"))
53
+ assert_nil io.close
54
+ assert_equal "PUTS!\nPUTZ\n", client.get_file_data("puts")
55
+
56
+ io = client.new_file("putc", :largefile => :stream)
57
+ assert_equal(0x20, io.putc(0x20))
58
+ assert_nil io.close
59
+ assert_equal " ", client.get_file_data("putc")
60
+
61
+ io = client.new_file("print splat", :largefile => :stream)
62
+ io.print(1, 2, 3)
63
+ assert_nil io.close
64
+ assert_equal "123", client.get_file_data("print splat")
65
+
66
+ io = client.new_file("printf", :largefile => :stream)
67
+ assert_nil io.printf("%x", 1638)
68
+ assert_nil io.close
69
+ assert_equal "666", client.get_file_data("printf")
70
+
71
+ io = client.new_file("syswrite", :largefile => :stream)
72
+ assert_equal 4, io.syswrite("good")
73
+ assert_equal 7, io.syswrite("morning")
74
+ assert_nil io.close
75
+ assert_equal "goodmorning", client.get_file_data("syswrite")
76
+
77
+ io = client.new_file("md5", :largefile=>:stream, :content_md5=>:trailer)
78
+ assert_instance_of Digest::MD5, io.md5
79
+ assert_nil io.puts("HIHI")
80
+ assert_nil io.close
81
+ assert_equal "HIHI\n", client.get_file_data("md5")
82
+ assert_equal Digest::MD5.hexdigest("HIHI\n"), io.md5.hexdigest
83
+
84
+ io = client.new_file("<<", :largefile=>:stream)
85
+ assert_equal(io, io << ">>")
86
+ assert_nil io.close
87
+ assert_equal ">>", client.get_file_data("<<")
88
+ end
89
+
90
+ def test_stream_new_file_with_content_length
91
+ add_host_device_domain
92
+ client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
93
+ io = client.new_file("clen", :largefile=>:stream,:content_length=>6)
94
+ io << "HIHIHI"
95
+ assert_nil io.close
96
+ assert_equal "HIHIHI", client.get_file_data("clen")
97
+
98
+ io = client.new_file("clen", :largefile=>:stream,:content_length=>1)
99
+ io << "FAIL"
100
+ assert_raises(MogileFS::SizeMismatchError) { io.close }
101
+ assert_equal "HIHIHI", client.get_file_data("clen")
102
+
103
+ io = client.new_file("md5", :largefile=>:stream,
104
+ :content_length=>6, :content_md5=>:trailer)
105
+ assert_equal(io, io << "MD5MD5")
106
+ assert_nil io.close
107
+ assert_equal "MD5MD5", client.get_file_data("md5")
108
+ assert_equal Digest::MD5.hexdigest("MD5MD5"), io.md5.hexdigest
109
+
110
+ io = client.new_file("md5", :largefile=>:stream,
111
+ :content_length=>6, :content_md5=>:trailer)
112
+ assert_equal(io, io << "MD5MD")
113
+ assert_raises(MogileFS::SizeMismatchError) { io.close }
114
+ assert_equal Digest::MD5.hexdigest("MD5MD"), io.md5.hexdigest
115
+ end
116
+
10
117
  def test_md5_check
11
118
  add_host_device_domain
12
119
  client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
@@ -23,6 +130,8 @@ class TestMogstoredRack < Test::Unit::TestCase
23
130
 
24
131
  def setup_mogstored
25
132
  @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
133
+ Dir.mkdir("#@docroot/dev1")
134
+ Dir.mkdir("#@docroot/dev2")
26
135
  @mogstored_mgmt = TCPServer.new(@test_host, 0)
27
136
  @mogstored_http = TCPServer.new(@test_host, 0)
28
137
  @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
@@ -47,12 +156,8 @@ EOF
47
156
 
48
157
  # I would use Rainbows! + *Threads + Ruby 1.9.3 in production
49
158
  def unicorn_setup
50
- examples_dir = Dir.pwd + "/examples"
51
- assert File.directory?(examples_dir)
52
159
  @ru = Tempfile.new(%w(mogstored_rack .ru))
53
160
  @ru.write <<EOF
54
- require "mogstored_rack"
55
- use Rack::Head
56
161
  run MogstoredRack.new("#@docroot")
57
162
  EOF
58
163
  @ru.flush
@@ -62,6 +167,7 @@ EOF
62
167
  @unicorn_stderr = Tempfile.new(%w(unicorn .stderr))
63
168
  @unicorn_stdout = Tempfile.new(%w(unicorn .stdout))
64
169
  @unicorn_conf.write <<EOF
170
+ require "mogstored_rack"
65
171
  listen "#@test_host:#{@mogstored_http_port}"
66
172
  pid "#{@unicorn_pid.path}"
67
173
  stderr_path "#{@unicorn_stderr.path}"
@@ -71,7 +177,7 @@ EOF
71
177
  @unicorn_conf.flush
72
178
 
73
179
  @mogstored_http.close
74
- x!("unicorn", "-I", examples_dir, "-E", "deployment",
180
+ x!("unicorn", "-E", "deployment",
75
181
  "--daemon", "--config", @unicorn_conf.path, @ru.path)
76
182
  wait_for_port @mogstored_http_port
77
183
  40.times do
@@ -86,4 +192,4 @@ EOF
86
192
  teardown_mogilefs
87
193
  puts(@unicorn_stderr.read) if $DEBUG
88
194
  end
89
- end if `which unicorn`.chomp.size > 0
195
+ end if ok && `which unicorn`.chomp.size > 0
data/test/test_pool.rb CHANGED
@@ -19,6 +19,22 @@ class ResourceWithArgs
19
19
  end
20
20
 
21
21
  end
22
+ class PoolClient < MogileFS::Client
23
+ attr_reader :alive
24
+
25
+ def initialize(*args)
26
+ @args = args
27
+ @alive = true
28
+ end
29
+
30
+ def backend
31
+ self
32
+ end
33
+
34
+ def shutdown
35
+ @alive = false
36
+ end
37
+ end
22
38
 
23
39
  class TestPool < Test::Unit::TestCase
24
40
 
@@ -95,5 +111,15 @@ class TestPool < Test::Unit::TestCase
95
111
  assert_equal o1, o2, "Objects must be reused"
96
112
  end
97
113
 
114
+ def test_auto_shutdown
115
+ pool = MogileFS::Pool.new(PoolClient, 666)
116
+ tmp = []
117
+ 6.times { tmp << pool.get }
118
+ tmp.each { |obj| pool.put(obj) }
119
+ alive = Hash.new { |h,k| h[k] = 0 }
120
+ tmp.each { |obj| alive[obj.alive] += 1 }
121
+ assert_equal 3, alive[true]
122
+ assert_equal 3, alive[false]
123
+ end
98
124
  end
99
125