mogilefs-client 2.2.0 → 3.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. data/.document +11 -0
  2. data/.gemtest +0 -0
  3. data/.gitignore +4 -0
  4. data/.wrongdoc.yml +5 -0
  5. data/GIT-VERSION-GEN +28 -0
  6. data/GNUmakefile +44 -0
  7. data/HACKING +33 -0
  8. data/{History.txt → History} +0 -1
  9. data/{LICENSE.txt → LICENSE} +0 -1
  10. data/Manifest.txt +34 -7
  11. data/README +51 -0
  12. data/Rakefile +11 -11
  13. data/TODO +10 -0
  14. data/bin/mog +109 -68
  15. data/examples/mogstored_rack.rb +189 -0
  16. data/lib/mogilefs.rb +56 -17
  17. data/lib/mogilefs/admin.rb +128 -62
  18. data/lib/mogilefs/backend.rb +205 -95
  19. data/lib/mogilefs/bigfile.rb +54 -70
  20. data/lib/mogilefs/bigfile/filter.rb +58 -0
  21. data/lib/mogilefs/chunker.rb +30 -0
  22. data/lib/mogilefs/client.rb +0 -2
  23. data/lib/mogilefs/copy_stream.rb +30 -0
  24. data/lib/mogilefs/http_file.rb +175 -0
  25. data/lib/mogilefs/http_reader.rb +79 -0
  26. data/lib/mogilefs/mogilefs.rb +242 -148
  27. data/lib/mogilefs/mysql.rb +3 -4
  28. data/lib/mogilefs/paths_size.rb +24 -0
  29. data/lib/mogilefs/pool.rb +0 -1
  30. data/lib/mogilefs/socket.rb +9 -0
  31. data/lib/mogilefs/socket/kgio.rb +55 -0
  32. data/lib/mogilefs/socket/pure_ruby.rb +70 -0
  33. data/lib/mogilefs/socket_common.rb +58 -0
  34. data/lib/mogilefs/util.rb +6 -169
  35. data/test/aggregate.rb +11 -11
  36. data/test/exec.rb +72 -0
  37. data/test/fresh.rb +222 -0
  38. data/test/integration.rb +43 -0
  39. data/test/setup.rb +1 -0
  40. data/test/socket_test.rb +98 -0
  41. data/test/test_admin.rb +14 -37
  42. data/test/test_backend.rb +50 -107
  43. data/test/test_bigfile.rb +2 -2
  44. data/test/test_db_backend.rb +1 -2
  45. data/test/test_fresh.rb +8 -0
  46. data/test/test_http_reader.rb +34 -0
  47. data/test/test_mogilefs.rb +278 -98
  48. data/test/test_mogilefs_integration.rb +174 -0
  49. data/test/test_mogilefs_integration_large_pipe.rb +62 -0
  50. data/test/test_mogilefs_integration_list_keys.rb +40 -0
  51. data/test/test_mogilefs_socket_kgio.rb +11 -0
  52. data/test/test_mogilefs_socket_pure.rb +7 -0
  53. data/test/test_mogstored_rack.rb +89 -0
  54. data/test/test_mogtool_bigfile.rb +116 -0
  55. data/test/test_mysql.rb +1 -2
  56. data/test/test_pool.rb +1 -1
  57. data/test/test_unit_mogstored_rack.rb +72 -0
  58. metadata +76 -54
  59. data/README.txt +0 -80
  60. data/lib/mogilefs/httpfile.rb +0 -157
  61. data/lib/mogilefs/network.rb +0 -107
  62. data/test/test_network.rb +0 -56
  63. data/test/test_util.rb +0 -121
@@ -1,107 +0,0 @@
1
- # -*- encoding: binary -*-
2
- require 'mogilefs'
3
- require 'socket'
4
- require 'mogilefs/util'
5
-
6
- module MogileFS::Network
7
- # given an array of URIs, verify that at least one of them is accessible
8
- # with the expected HTTP code within the timeout period (in seconds).
9
- def verify_uris(uris = [], expect = '200', timeout = 2.00)
10
- uri_socks = {}
11
-
12
- # first, we asynchronously connect to all of them
13
- uris.each do |uri|
14
- sock = Socket.mogilefs_new_nonblock(uri.host, uri.port) rescue next
15
- uri_socks[sock] = uri
16
- end
17
-
18
- # wait for at least one of them to finish connecting and send
19
- # HTTP requests to the connected ones
20
- sockets, timeout = get_writable_set(uri_socks, timeout)
21
-
22
- # Await a response from the sockets we had written to, we only need one
23
- # valid response, but we'll take more if they return simultaneously
24
- sockets[0] ? get_readable_uris(sockets, uri_socks, expect, timeout) : []
25
-
26
- ensure
27
- uri_socks.keys.each { |sock| sock.close rescue nil }
28
- end
29
-
30
- private
31
- include MogileFS::Util
32
-
33
- # returns an array of writeable Sockets and leftover from timeout
34
- def get_writable_set(uri_socks, timeout)
35
- sockets = []
36
- begin
37
- t0 = Time.now
38
- r = begin
39
- IO.select(nil, uri_socks.keys, nil, timeout > 0 ? timeout : 0)
40
- rescue
41
- # get rid of bad descriptors
42
- uri_socks.delete_if do |sock, uri|
43
- begin
44
- sock.recv_nonblock(1)
45
- false # should never get here for HTTP, really...
46
- rescue Errno::EAGAIN, Errno::EINTR
47
- false
48
- rescue
49
- sock.close rescue nil
50
- true
51
- end
52
- end
53
- timeout -= (Time.now - t0)
54
- retry if timeout >= 0
55
- end
56
-
57
- break unless r && r[1]
58
-
59
- r[1].each do |sock|
60
- begin
61
- # we don't care about short/interrupted writes here, if the
62
- # following request fails or blocks then the server is
63
- # flat-out hopeless
64
- sock.write_nonblock(
65
- "HEAD #{uri_socks[sock].request_uri} HTTP/1.0\r\n\r\n")
66
- sockets << sock
67
- rescue
68
- sock.close rescue nil
69
- end
70
- end
71
-
72
- timeout -= (Time.now - t0)
73
- end until (sockets[0] || timeout < 0)
74
-
75
- [ sockets, timeout ]
76
- end
77
-
78
- # returns an array of URIs from uri_socks that are good
79
- def get_readable_uris(sockets, uri_socks, expect, timeout)
80
- ok_uris = []
81
-
82
- begin
83
- t0 = Time.now
84
- r = IO.select(sockets, nil, nil, timeout > 0 ? timeout : 0) rescue nil
85
-
86
- (r && r[0] ? r[0] : sockets).each do |sock|
87
- buf = begin
88
- sock.recv_nonblock(128, Socket::MSG_PEEK)
89
- rescue Errno::EAGAIN, Errno::EINTR
90
- next
91
- rescue
92
- sockets.delete(sock) # socket went bad
93
- next
94
- end
95
-
96
- if buf && /\AHTTP\/[\d\.]+ #{expect} / =~ buf
97
- ok_uris << uri_socks.delete(sock)
98
- sock.close rescue nil
99
- end
100
- end
101
- timeout -= (Time.now - t0)
102
- end until ok_uris[0] || timeout < 0
103
-
104
- ok_uris
105
- end
106
-
107
- end # module MogileFS::Network
data/test/test_network.rb DELETED
@@ -1,56 +0,0 @@
1
- # -*- encoding: binary -*-
2
- require 'test/setup'
3
- require 'mogilefs'
4
- require 'mogilefs/network'
5
-
6
- class TestNetwork < Test::Unit::TestCase
7
- include MogileFS::Network
8
-
9
- def test_verify_uris
10
- good = TempServer.new(Proc.new do |serv,port|
11
- client,client_addr = serv.accept
12
- client.readpartial(4096)
13
- client.syswrite("HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n")
14
- end)
15
- bad = TempServer.new(Proc.new do |serv,port|
16
- client, client_addr = serv.accept
17
- client.close rescue nil
18
- end)
19
-
20
- good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
21
- bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
22
- ok = verify_uris([ good_uri, bad_uri ])
23
- assert_equal [ good_uri ], ok
24
- ensure
25
- TempServer.destroy_all!
26
- end
27
-
28
- def test_verify_non_existent
29
- good = TempServer.new(Proc.new do |serv,port|
30
- client,client_addr = serv.accept
31
- client.readpartial(4096)
32
- client.syswrite("HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n")
33
- end)
34
- bad = TempServer.new(Proc.new { |serv,port| serv.close })
35
-
36
- good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
37
- bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
38
- ok = verify_uris([ good_uri, bad_uri ])
39
- assert_equal [ good_uri ], ok
40
- ensure
41
- TempServer.destroy_all!
42
- end
43
-
44
- def test_verify_all_bad
45
- good = TempServer.new(Proc.new { |serv,port| serv.close })
46
- bad = TempServer.new(Proc.new { |serv,port| serv.close })
47
-
48
- good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
49
- bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
50
- ok = verify_uris([ good_uri, bad_uri ], '200', 1.0)
51
- assert ok.empty?, "nothing returned"
52
- ensure
53
- TempServer.destroy_all!
54
- end
55
-
56
- end
data/test/test_util.rb DELETED
@@ -1,121 +0,0 @@
1
- # -*- encoding: binary -*-
2
- require 'test/setup'
3
-
4
- class TestMogileFS__Util < Test::Unit::TestCase
5
- include MogileFS::Util
6
-
7
- def test_mogilefs_write
8
- rd, wr = IO.pipe
9
-
10
- svr = Proc.new do |serv, port|
11
- client, client_addr = serv.accept
12
- client.sync = true
13
- nr = 0
14
- loop do
15
- begin
16
- nr += client.readpartial(16384).length
17
- rescue EOFError
18
- break
19
- end
20
- end
21
- wr.syswrite("#{nr}\n")
22
- client.close rescue nil
23
- end
24
- t = TempServer.new(svr)
25
- s = Socket.mogilefs_new('127.0.0.1', t.port)
26
- tmp = s.getsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF)
27
- sndbuf_bytes = tmp.unpack('i')[0]
28
- big_string = ' ' * (sndbuf_bytes * 10)
29
-
30
- sent = s.send(big_string, 0)
31
- assert(sent < big_string.length)
32
-
33
- syswrite_full(s, big_string)
34
- s.close rescue nil
35
- IO.select([rd])
36
- assert_equal((sent + big_string.length), rd.sysread(4096).to_i)
37
- ensure
38
- t.destroy!
39
- end
40
-
41
- def test_write_timeout
42
- svr = Proc.new do |serv, port|
43
- client, client_addr = serv.accept
44
- client.sync = true
45
- readed = client.readpartial(16384)
46
- sleep
47
- end
48
- t = TempServer.new(svr)
49
- s = Socket.mogilefs_new('127.0.0.1', t.port)
50
- tmp = s.getsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF)
51
- sndbuf_bytes = tmp.unpack('i')[0]
52
- big_string = ' ' * (sndbuf_bytes * 10)
53
-
54
- assert_raises(MogileFS::Timeout) { syswrite_full(s, big_string, 0.1) }
55
- s.close rescue nil
56
- ensure
57
- t.destroy!
58
- end
59
-
60
- def test_sysread_slowly
61
- nr = 10
62
- str = 'abcde'
63
- expect = str * nr
64
- rd, wr = IO.pipe
65
- pid = fork do
66
- rd.close
67
- nr.times do
68
- syswrite_full(wr, str)
69
- sleep(0.1)
70
- end
71
- end
72
- wr.close
73
- buf = sysread_full(rd, expect.size)
74
- assert_equal expect, buf
75
- rd.close
76
- ensure
77
- Process.kill('TERM', pid) rescue nil
78
- Process.waitpid(pid) rescue nil
79
- end
80
-
81
- def test_sysread_timeout
82
- nr = 10
83
- str = 'abcde'
84
- expect = str * nr
85
- rd, wr = IO.pipe
86
- pid = fork do
87
- rd.close
88
- nr.times do
89
- syswrite_full(wr, str)
90
- sleep 1
91
- end
92
- end
93
- wr.close
94
- assert_raises(MogileFS::Timeout) { sysread_full(rd, expect.size, 0.1) }
95
- rd.close
96
- ensure
97
- Process.kill('TERM', pid) rescue nil
98
- Process.waitpid(pid) rescue nil
99
- end
100
-
101
- def test_sysread_full_timeout
102
- nr = 100
103
- str = 'abcde'
104
- expect = str * nr
105
- rd, wr = IO.pipe
106
- pid = fork do
107
- rd.close
108
- nr.times do
109
- syswrite_full(wr, str)
110
- sleep 0.01
111
- end
112
- end
113
- wr.close
114
- assert_raises(MogileFS::Timeout) { sysread_full(rd,expect.size,0.1,true) }
115
- rd.close
116
- ensure
117
- Process.kill('TERM', pid) rescue nil
118
- Process.waitpid(pid) rescue nil
119
- end
120
-
121
- end