mogilefs-client 2.2.0 → 3.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. data/.document +11 -0
  2. data/.gemtest +0 -0
  3. data/.gitignore +4 -0
  4. data/.wrongdoc.yml +5 -0
  5. data/GIT-VERSION-GEN +28 -0
  6. data/GNUmakefile +44 -0
  7. data/HACKING +33 -0
  8. data/{History.txt → History} +0 -1
  9. data/{LICENSE.txt → LICENSE} +0 -1
  10. data/Manifest.txt +34 -7
  11. data/README +51 -0
  12. data/Rakefile +11 -11
  13. data/TODO +10 -0
  14. data/bin/mog +109 -68
  15. data/examples/mogstored_rack.rb +189 -0
  16. data/lib/mogilefs.rb +56 -17
  17. data/lib/mogilefs/admin.rb +128 -62
  18. data/lib/mogilefs/backend.rb +205 -95
  19. data/lib/mogilefs/bigfile.rb +54 -70
  20. data/lib/mogilefs/bigfile/filter.rb +58 -0
  21. data/lib/mogilefs/chunker.rb +30 -0
  22. data/lib/mogilefs/client.rb +0 -2
  23. data/lib/mogilefs/copy_stream.rb +30 -0
  24. data/lib/mogilefs/http_file.rb +175 -0
  25. data/lib/mogilefs/http_reader.rb +79 -0
  26. data/lib/mogilefs/mogilefs.rb +242 -148
  27. data/lib/mogilefs/mysql.rb +3 -4
  28. data/lib/mogilefs/paths_size.rb +24 -0
  29. data/lib/mogilefs/pool.rb +0 -1
  30. data/lib/mogilefs/socket.rb +9 -0
  31. data/lib/mogilefs/socket/kgio.rb +55 -0
  32. data/lib/mogilefs/socket/pure_ruby.rb +70 -0
  33. data/lib/mogilefs/socket_common.rb +58 -0
  34. data/lib/mogilefs/util.rb +6 -169
  35. data/test/aggregate.rb +11 -11
  36. data/test/exec.rb +72 -0
  37. data/test/fresh.rb +222 -0
  38. data/test/integration.rb +43 -0
  39. data/test/setup.rb +1 -0
  40. data/test/socket_test.rb +98 -0
  41. data/test/test_admin.rb +14 -37
  42. data/test/test_backend.rb +50 -107
  43. data/test/test_bigfile.rb +2 -2
  44. data/test/test_db_backend.rb +1 -2
  45. data/test/test_fresh.rb +8 -0
  46. data/test/test_http_reader.rb +34 -0
  47. data/test/test_mogilefs.rb +278 -98
  48. data/test/test_mogilefs_integration.rb +174 -0
  49. data/test/test_mogilefs_integration_large_pipe.rb +62 -0
  50. data/test/test_mogilefs_integration_list_keys.rb +40 -0
  51. data/test/test_mogilefs_socket_kgio.rb +11 -0
  52. data/test/test_mogilefs_socket_pure.rb +7 -0
  53. data/test/test_mogstored_rack.rb +89 -0
  54. data/test/test_mogtool_bigfile.rb +116 -0
  55. data/test/test_mysql.rb +1 -2
  56. data/test/test_pool.rb +1 -1
  57. data/test/test_unit_mogstored_rack.rb +72 -0
  58. metadata +76 -54
  59. data/README.txt +0 -80
  60. data/lib/mogilefs/httpfile.rb +0 -157
  61. data/lib/mogilefs/network.rb +0 -107
  62. data/test/test_network.rb +0 -56
  63. data/test/test_util.rb +0 -121
@@ -0,0 +1,174 @@
1
+ # -*- encoding: binary -*-
2
+ require './test/integration'
3
+
4
+ class TestMogileFSIntegration < TestMogIntegration
5
+ def setup
6
+ super
7
+ @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
8
+ end
9
+
10
+ def test_CRUD
11
+ assert_equal 4, @client.store_content("CRUD", "default", "DATA")
12
+ assert_equal 4, @client.size("CRUD")
13
+ assert_equal "DATA", @client.get_file_data("CRUD")
14
+ assert_equal "DAT", @client.get_file_data("CRUD", nil, 3)
15
+ assert_equal "AT", @client.get_file_data("CRUD", nil, 2, 1)
16
+
17
+ tmp = Tempfile.new("z")
18
+ tmp_path = tmp.path
19
+ tmp.close!
20
+ assert_equal 4, @client.get_file_data("CRUD", tmp_path)
21
+ assert_equal "DATA", File.read(tmp_path)
22
+ st = File.stat(tmp_path)
23
+ assert_equal 0100600, st.mode
24
+ File.unlink(tmp_path)
25
+
26
+ sio = StringIO.new("")
27
+ rv = @client.get_file_data("CRUD", sio)
28
+ assert_equal 4, rv
29
+ assert_equal "DATA", sio.string
30
+ assert_equal 8, @client.store_content("CRUD", "default", "MOARDATA")
31
+ assert_equal "MOARDATA", @client.get_file_data("CRUD")
32
+ assert_equal true, @client.delete("CRUD")
33
+ assert_raises(MogileFS::Backend::UnknownKeyError) { @client.delete("CRUD") }
34
+
35
+ data = "hello world\n".freeze
36
+ tmp = tmpfile("blob")
37
+ tmp.sync = true
38
+ tmp.write(data)
39
+ tmp.rewind
40
+ assert_equal tmp.size, @client.store_file("blob", nil, tmp)
41
+ assert_equal(data, @client.get_file_data("blob"))
42
+
43
+ data = "pipe!\n".freeze
44
+ r, w = IO.pipe
45
+ th = Thread.new do
46
+ w.write(data)
47
+ w.close
48
+ end
49
+ assert_equal data.size, @client.store_file("pipe", nil, r)
50
+ assert_nothing_raised do
51
+ r.close
52
+ th.join
53
+ end
54
+ assert_equal(data, @client.get_file_data("pipe"))
55
+
56
+ cbk = MogileFS::Util::StoreContent.new(nil) do |write_callback|
57
+ 10.times { write_callback.call("data") }
58
+ end
59
+ assert_nil cbk.length
60
+ nr = @client.store_content('store_content', nil, cbk)
61
+ assert_equal 40, nr
62
+ assert_equal("data" * 10, @client.get_file_data('store_content'))
63
+ end
64
+
65
+ def test_store_non_rewindable
66
+ tmp = Object.new
67
+ def tmp.size
68
+ 666
69
+ end
70
+
71
+ assert_raises(MogileFS::HTTPFile::NonRetryableError) do
72
+ @client.store_file("non_rewindable", nil, tmp)
73
+ end
74
+ end
75
+
76
+ def test_file_info
77
+ assert_equal 3, @client.store_content("file_info", "default", "FOO")
78
+ res = @client.file_info("file_info")
79
+ assert_kind_of Integer, res["fid"]
80
+ assert_equal 3, res["length"]
81
+ assert ! res.include?("devids")
82
+ assert_kind_of Integer, res["devcount"]
83
+
84
+ res = @client.file_info("file_info", :devices => true)
85
+ assert_kind_of Integer, res["fid"]
86
+ assert_equal 3, res["length"]
87
+ assert_kind_of Integer, res["devcount"]
88
+ devids = res.delete("devids")
89
+ assert_instance_of Array, devids
90
+ devids.each { |devid| assert_kind_of Integer, devid }
91
+ assert_equal res["devcount"], devids.size
92
+ end
93
+
94
+ def test_file_debug
95
+ assert_equal 3, @client.store_content("file_debug", "default", "BUG")
96
+ a = @client.file_debug("file_debug")
97
+ b = @client.file_debug(:key => "file_debug")
98
+ fid = @client.file_info("file_debug")["fid"]
99
+ c = @client.file_debug(fid)
100
+ d = @client.file_debug(:fid => fid)
101
+
102
+ [ a, b, c, d ].each do |res|
103
+ assert_equal fid, res["fid_fid"]
104
+ assert_equal 0, res["fid_classid"]
105
+ assert_equal "file_debug", res["fid_dkey"]
106
+ assert_equal 3, res["fid_length"]
107
+ assert_kind_of Array, res["devids"]
108
+ assert_kind_of Integer, res["devids"][0]
109
+ res["devids"].each do |devid|
110
+ uri = URI.parse(res["devpath_#{devid}"])
111
+ assert_equal "http", uri.scheme
112
+ end
113
+ assert_equal "default", res["fid_class"]
114
+ end
115
+ @client.delete("file_debug")
116
+ rv = @client.file_debug(fid)
117
+ assert rv.keys.grep(/\Afid_/).empty?, rv.inspect
118
+ end
119
+
120
+ def test_file_debug_in_progress
121
+ rv = @client.new_file("file_debug_in_progress") do |http_file|
122
+ http_file << "ZZZZ"
123
+ dests = http_file.instance_variable_get(:@dests)
124
+ dests[0][1] =~ %r{/(\d+)\.fid\z}
125
+ fid = $1.to_i
126
+ rv = @client.file_debug(fid)
127
+ devids = dests.map { |x| x[0].to_i }.sort
128
+ assert_equal devids, rv["tempfile_devids"].sort
129
+ assert_equal "file_debug_in_progress", rv["tempfile_dkey"]
130
+ end
131
+ assert_equal 4, rv
132
+ end
133
+
134
+ def test_admin_get_devices
135
+ admin = MogileFS::Admin.new(:hosts => @trackers)
136
+ devices = admin.get_devices
137
+ if any_device = devices[0]
138
+ %w(mb_asof mb_free mb_used mb_total devid weight hostid).each do |field|
139
+ case value = any_device[field]
140
+ when nil
141
+ when Integer
142
+ assert value >= 0, "#{field}=#{value.inspect} is negative"
143
+ else
144
+ assert false, "#{field}=#{value.inspect} is #{value.class}"
145
+ end
146
+ end
147
+
148
+ field = "utilization"
149
+ case value = any_device[field]
150
+ when nil
151
+ when Float
152
+ assert value >= 0.0, "#{field}=#{value.inspect} is negative"
153
+ else
154
+ assert false, "#{field}=#{value.inspect} is #{value.class}"
155
+ end
156
+ end
157
+ end
158
+
159
+ # TODO: move this to a fresh instance
160
+ def test_admin_each_fid
161
+ admin = MogileFS::Admin.new(:hosts => @trackers)
162
+ seen = {}
163
+ count = admin.each_fid do |info|
164
+ seen[info["fid"]] = true
165
+ assert_kind_of Integer, info["fid"]
166
+ assert_kind_of Integer, info["length"]
167
+ assert_kind_of Integer, info["devcount"]
168
+ assert_kind_of String, info["key"]
169
+ assert_kind_of String, info["class"]
170
+ assert_kind_of String, info["domain"]
171
+ end
172
+ assert_equal count, seen.size
173
+ end if ENV["TEST_EXPENSIVE"]
174
+ end
@@ -0,0 +1,62 @@
1
+ # -*- encoding: binary -*-
2
+ require './test/integration'
3
+ require "digest/sha1"
4
+
5
+ class TestMogileFSLargePipe< TestMogIntegration
6
+ def setup
7
+ super
8
+ @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
9
+ end
10
+
11
+ def test_large_pipe_test
12
+ junk = File.open("/dev/urandom") { |fp| fp.read(1024) }
13
+ junk *= 32
14
+ nr = rand(666) + 1024
15
+ r, w = IO.pipe
16
+ sha1 = Digest::SHA1.new
17
+ th = Thread.new do
18
+ nr.times do
19
+ sha1.update(junk)
20
+ w.write(junk)
21
+ end
22
+ w.close
23
+ end
24
+ assert_equal(nr * junk.size, @client.store_file("a", nil, r))
25
+ r.close
26
+ th.join
27
+ @client.get_file_data("a") do |rd|
28
+ assert_equal(nr * junk.size, @client.store_file("b", nil, rd))
29
+ end
30
+ a = Thread.new { @client.get_file_data("a") { |rd| sha1read(rd) } }
31
+ b = Thread.new { @client.get_file_data("b") { |rd| sha1read(rd) } }
32
+ a = a.value
33
+ b = b.value
34
+ assert_equal a, b
35
+ assert_equal sha1, a
36
+
37
+ # We should be able to open FIFOs
38
+ tmp = tmpfile("fifo")
39
+ tmp_path = tmp.path
40
+ File.unlink(tmp_path)
41
+ x!("mkfifo", tmp_path)
42
+ pid = fork do
43
+ File.open(tmp_path, "wb") do |wr|
44
+ nr.times { wr.write(junk) }
45
+ end
46
+ end
47
+ assert_equal(nr * junk.size, @client.store_file("fifo", nil, tmp_path))
48
+ _, status = Process.waitpid2(pid)
49
+ assert status.success?, status.inspect
50
+ fifo_sha1 = @client.get_file_data("fifo") { |rd| sha1read(rd) }
51
+ assert_equal sha1, fifo_sha1
52
+ end
53
+
54
+ def sha1read(rd)
55
+ buf = ""
56
+ d = Digest::SHA1.new
57
+ while rd.read(16384, buf)
58
+ d << buf
59
+ end
60
+ d
61
+ end
62
+ end
@@ -0,0 +1,40 @@
1
+
2
+ # -*- encoding: binary -*-
3
+ require './test/integration'
4
+
5
+ class TestMogileFSIntegrationListKeys < TestMogIntegration
6
+ def setup
7
+ super
8
+ @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
9
+ end
10
+
11
+ def test_list_keys
12
+ k = %w(a b c d e f g)
13
+ k.each { |x| @client.store_content("lk_#{x}", nil, x) }
14
+ expect = k.map { |x| "lk_#{x}" }
15
+ rv = @client.list_keys
16
+ assert_equal([ expect, expect.last ] , rv)
17
+ nr = 0
18
+ @client.list_keys do |key, length, devcount|
19
+ assert_equal 1, length
20
+ assert_kind_of Integer, devcount
21
+ assert_equal expect[nr], key
22
+ nr += 1
23
+ end
24
+ end
25
+
26
+ def test_list_keys_strange
27
+ @client.store_content("hello+world", nil, "HI")
28
+ rv = @client.list_keys
29
+ assert_equal "hello+world", rv[0][0]
30
+ end
31
+
32
+ def test_each_key
33
+ 9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) }
34
+ n = 0
35
+ @client.each_key do |key|
36
+ assert_equal "ek_#{n.to_s}", key
37
+ n += 1
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,11 @@
1
+ require "./test/socket_test"
2
+ require "mogilefs"
3
+ begin
4
+ require "kgio"
5
+ require "mogilefs/socket/kgio"
6
+ rescue LoadError
7
+ end unless ENV["MOGILEFS_CLIENT_PURE"]
8
+
9
+ class TestSocketKgio < Test::Unit::TestCase
10
+ include SocketTest
11
+ end if defined?(Kgio)
@@ -0,0 +1,7 @@
1
+ ENV["MOGILEFS_CLIENT_PURE"] = "true"
2
+ require "./test/socket_test"
3
+ require "mogilefs"
4
+
5
+ class TestSocketPure < Test::Unit::TestCase
6
+ include SocketTest
7
+ end
@@ -0,0 +1,89 @@
1
+ # -*- encoding: binary -*-
2
+ require "./test/fresh"
3
+
4
+ class TestMogstoredRack < Test::Unit::TestCase
5
+ include TestFreshSetup
6
+ def setup
7
+ setup_mogilefs
8
+ end
9
+
10
+ def test_md5_check
11
+ add_host_device_domain
12
+ client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
13
+ node = "#@test_host:#@mogstored_http_port"
14
+ pid = fork do
15
+ # not modifying this hash in the same process
16
+ MogileFS::HTTPFile::MD5_TRAILER_NODES[node] = true
17
+ client.store_content("md5_me", nil, "HELLO WORLD")
18
+ end
19
+ _, status = Process.waitpid2(pid)
20
+ assert status.success?, status.inspect
21
+ assert_equal "HELLO WORLD", client.get_file_data("md5_me")
22
+ end
23
+
24
+ def setup_mogstored
25
+ @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
26
+ @mogstored_mgmt = TCPServer.new(@test_host, 0)
27
+ @mogstored_http = TCPServer.new(@test_host, 0)
28
+ @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
29
+ @mogstored_http_port = @mogstored_http.addr[1]
30
+ @mogstored_conf = Tempfile.new(["mogstored", "conf"])
31
+ @mogstored_pid = Tempfile.new(["mogstored", "pid"])
32
+ @mogstored_conf.write <<EOF
33
+ pidfile = #{@mogstored_pid.path}
34
+ maxconns = 1000
35
+ mgmtlisten = #@test_host:#{@mogstored_mgmt_port}
36
+ server = none
37
+ docroot = #@docroot
38
+ EOF
39
+ @mogstored_conf.flush
40
+ @mogstored_mgmt.close
41
+
42
+ unicorn_setup
43
+
44
+ x!("mogstored", "--daemon", "--config=#{@mogstored_conf.path}")
45
+ wait_for_port @mogstored_mgmt_port
46
+ end
47
+
48
+ # I would use Rainbows! + *Threads + Ruby 1.9.3 in production
49
+ def unicorn_setup
50
+ examples_dir = Dir.pwd + "/examples"
51
+ assert File.directory?(examples_dir)
52
+ @ru = Tempfile.new(%w(mogstored_rack .ru))
53
+ @ru.write <<EOF
54
+ require "mogstored_rack"
55
+ use Rack::Head
56
+ run MogstoredRack.new("#@docroot")
57
+ EOF
58
+ @ru.flush
59
+
60
+ @unicorn_pid = Tempfile.new(%w(unicorn .pid))
61
+ @unicorn_conf = Tempfile.new(%w(unicorn.conf .rb))
62
+ @unicorn_stderr = Tempfile.new(%w(unicorn .stderr))
63
+ @unicorn_stdout = Tempfile.new(%w(unicorn .stdout))
64
+ @unicorn_conf.write <<EOF
65
+ listen "#@test_host:#{@mogstored_http_port}"
66
+ pid "#{@unicorn_pid.path}"
67
+ stderr_path "#{@unicorn_stderr.path}"
68
+ stdout_path "#{@unicorn_stdout.path}"
69
+ rewindable_input false
70
+ EOF
71
+ @unicorn_conf.flush
72
+
73
+ @mogstored_http.close
74
+ x!("unicorn", "-I", examples_dir, "-E", "deployment",
75
+ "--daemon", "--config", @unicorn_conf.path, @ru.path)
76
+ wait_for_port @mogstored_http_port
77
+ 40.times do
78
+ break if File.size(@unicorn_pid.path) > 0
79
+ sleep 0.1
80
+ end
81
+ end
82
+
83
+ def teardown
84
+ pid = File.read(@unicorn_pid.path).to_i
85
+ Process.kill(:QUIT, pid) if pid > 0
86
+ teardown_mogilefs
87
+ puts(@unicorn_stderr.read) if $DEBUG
88
+ end
89
+ end if `which unicorn`.chomp.size > 0
@@ -0,0 +1,116 @@
1
+ # -*- encoding: binary -*-
2
+ require "./test/integration"
3
+ require "net/http"
4
+ ok = true
5
+ unless File.executable?(`which mogtool 2>/dev/null`.strip)
6
+ warn "mogtool not found, skipping #{__FILE__}"
7
+ ok = false
8
+ end
9
+
10
+ class TestMogtoolBigfile < TestMogIntegration
11
+ buf = File.open("/dev/urandom") { |fp| fp.read(1024) }
12
+ buf *= 1024
13
+ RAND = Tempfile.new("rand")
14
+ RAND.sync = true
15
+ sha1 = Digest::SHA1.new
16
+ 100.times { sha1 << buf; RAND.write(buf) }
17
+ buf = nil
18
+ RAND_SHA1 = sha1.hexdigest
19
+
20
+ def setup
21
+ super
22
+ RAND.rewind
23
+ @big_uuid = "big-#{uuid}"
24
+ @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
25
+ end
26
+
27
+ def mogtool!(*args)
28
+ x!("mogtool", "--trackers=#{@trackers.join(',')}",
29
+ "--domain=#@domain", *args)
30
+ end
31
+
32
+ # the mogtool definition of gzip is wrong and just raw zlib deflate
33
+ def test_bigfile_gzip_mogtool
34
+ mogtool!("inject", "--gzip", "--bigfile", RAND.path, @big_uuid)
35
+ sha1_check
36
+ end
37
+
38
+ def test_bigfile_mogtool
39
+ mogtool!("inject", "--bigfile", RAND.path, @big_uuid)
40
+ sha1_check
41
+
42
+ # ensure fallback works for rebalanced/replaced files
43
+ part1 = "#@big_uuid,1"
44
+ tmp = tmpfile("part1")
45
+ before_uris = @client.get_uris(part1)
46
+ @client.get_file_data(part1, tmp)
47
+ @client.delete(part1)
48
+ @client.store_file(part1, nil, tmp.path)
49
+ wait_for_DELETE(before_uris)
50
+ sha1_check
51
+
52
+ # corrupt the existing data in part1
53
+ @client.store_content(part1, nil, "HELLO")
54
+ assert_nothing_raised { @client.get_uris(part1) }
55
+
56
+ # corruption is detected on verify
57
+ junk = tmpfile("junk")
58
+ assert_raises(MogileFS::ChecksumMismatchError) do
59
+ @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
60
+ end
61
+
62
+ # corruption is NOT detected on verify
63
+ junk = tmpfile("junk")
64
+ assert_nothing_raised do
65
+ @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => false)
66
+ end
67
+
68
+ # restoring no-corrupted data succeeds!
69
+ @client.store_file(part1, nil, tmp.path)
70
+ sha1_check
71
+
72
+ # missing parts fail
73
+ before_uris = @client.get_uris(part1)
74
+ @client.delete(part1)
75
+ junk = tmpfile("junk")
76
+ assert_raises(MogileFS::Backend::UnknownKeyError) do
77
+ @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
78
+ end
79
+ end
80
+
81
+ def wait_for_DELETE(uris)
82
+ uris.each do |uri|
83
+ tries = 0
84
+ begin
85
+ Net::HTTP.start(uri.host, uri.port) do |http|
86
+ sleep(0.1) while Net::HTTPOK === http.head(uri.path)
87
+ end
88
+ rescue
89
+ if (tries += 1) < 666
90
+ sleep(0.1)
91
+ retry
92
+ end
93
+ raise
94
+ end
95
+ end
96
+ end
97
+
98
+ def sha1_check
99
+ r, w = IO.pipe
100
+ @to_close << r
101
+ @to_close << w
102
+ th = Thread.new do
103
+ sha1 = Digest::SHA1.new
104
+ buf = ""
105
+ while r.read(16384, buf)
106
+ sha1 << buf
107
+ end
108
+ sha1.hexdigest
109
+ end
110
+ res = @client.bigfile_write("_big_info:#@big_uuid", w, :verify => true)
111
+ w.close
112
+ read_sha1 = th.value
113
+ assert_equal RAND_SHA1, read_sha1
114
+ assert_equal RAND.size, res[0]
115
+ end
116
+ end if ok