mogilefs-client 2.2.0 → 3.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +11 -0
- data/.gemtest +0 -0
- data/.gitignore +4 -0
- data/.wrongdoc.yml +5 -0
- data/GIT-VERSION-GEN +28 -0
- data/GNUmakefile +44 -0
- data/HACKING +33 -0
- data/{History.txt → History} +0 -1
- data/{LICENSE.txt → LICENSE} +0 -1
- data/Manifest.txt +34 -7
- data/README +51 -0
- data/Rakefile +11 -11
- data/TODO +10 -0
- data/bin/mog +109 -68
- data/examples/mogstored_rack.rb +189 -0
- data/lib/mogilefs.rb +56 -17
- data/lib/mogilefs/admin.rb +128 -62
- data/lib/mogilefs/backend.rb +205 -95
- data/lib/mogilefs/bigfile.rb +54 -70
- data/lib/mogilefs/bigfile/filter.rb +58 -0
- data/lib/mogilefs/chunker.rb +30 -0
- data/lib/mogilefs/client.rb +0 -2
- data/lib/mogilefs/copy_stream.rb +30 -0
- data/lib/mogilefs/http_file.rb +175 -0
- data/lib/mogilefs/http_reader.rb +79 -0
- data/lib/mogilefs/mogilefs.rb +242 -148
- data/lib/mogilefs/mysql.rb +3 -4
- data/lib/mogilefs/paths_size.rb +24 -0
- data/lib/mogilefs/pool.rb +0 -1
- data/lib/mogilefs/socket.rb +9 -0
- data/lib/mogilefs/socket/kgio.rb +55 -0
- data/lib/mogilefs/socket/pure_ruby.rb +70 -0
- data/lib/mogilefs/socket_common.rb +58 -0
- data/lib/mogilefs/util.rb +6 -169
- data/test/aggregate.rb +11 -11
- data/test/exec.rb +72 -0
- data/test/fresh.rb +222 -0
- data/test/integration.rb +43 -0
- data/test/setup.rb +1 -0
- data/test/socket_test.rb +98 -0
- data/test/test_admin.rb +14 -37
- data/test/test_backend.rb +50 -107
- data/test/test_bigfile.rb +2 -2
- data/test/test_db_backend.rb +1 -2
- data/test/test_fresh.rb +8 -0
- data/test/test_http_reader.rb +34 -0
- data/test/test_mogilefs.rb +278 -98
- data/test/test_mogilefs_integration.rb +174 -0
- data/test/test_mogilefs_integration_large_pipe.rb +62 -0
- data/test/test_mogilefs_integration_list_keys.rb +40 -0
- data/test/test_mogilefs_socket_kgio.rb +11 -0
- data/test/test_mogilefs_socket_pure.rb +7 -0
- data/test/test_mogstored_rack.rb +89 -0
- data/test/test_mogtool_bigfile.rb +116 -0
- data/test/test_mysql.rb +1 -2
- data/test/test_pool.rb +1 -1
- data/test/test_unit_mogstored_rack.rb +72 -0
- metadata +76 -54
- data/README.txt +0 -80
- data/lib/mogilefs/httpfile.rb +0 -157
- data/lib/mogilefs/network.rb +0 -107
- data/test/test_network.rb +0 -56
- data/test/test_util.rb +0 -121
data/lib/mogilefs/network.rb
DELETED
@@ -1,107 +0,0 @@
|
|
1
|
-
# -*- encoding: binary -*-
|
2
|
-
require 'mogilefs'
|
3
|
-
require 'socket'
|
4
|
-
require 'mogilefs/util'
|
5
|
-
|
6
|
-
module MogileFS::Network
|
7
|
-
# given an array of URIs, verify that at least one of them is accessible
|
8
|
-
# with the expected HTTP code within the timeout period (in seconds).
|
9
|
-
def verify_uris(uris = [], expect = '200', timeout = 2.00)
|
10
|
-
uri_socks = {}
|
11
|
-
|
12
|
-
# first, we asynchronously connect to all of them
|
13
|
-
uris.each do |uri|
|
14
|
-
sock = Socket.mogilefs_new_nonblock(uri.host, uri.port) rescue next
|
15
|
-
uri_socks[sock] = uri
|
16
|
-
end
|
17
|
-
|
18
|
-
# wait for at least one of them to finish connecting and send
|
19
|
-
# HTTP requests to the connected ones
|
20
|
-
sockets, timeout = get_writable_set(uri_socks, timeout)
|
21
|
-
|
22
|
-
# Await a response from the sockets we had written to, we only need one
|
23
|
-
# valid response, but we'll take more if they return simultaneously
|
24
|
-
sockets[0] ? get_readable_uris(sockets, uri_socks, expect, timeout) : []
|
25
|
-
|
26
|
-
ensure
|
27
|
-
uri_socks.keys.each { |sock| sock.close rescue nil }
|
28
|
-
end
|
29
|
-
|
30
|
-
private
|
31
|
-
include MogileFS::Util
|
32
|
-
|
33
|
-
# returns an array of writeable Sockets and leftover from timeout
|
34
|
-
def get_writable_set(uri_socks, timeout)
|
35
|
-
sockets = []
|
36
|
-
begin
|
37
|
-
t0 = Time.now
|
38
|
-
r = begin
|
39
|
-
IO.select(nil, uri_socks.keys, nil, timeout > 0 ? timeout : 0)
|
40
|
-
rescue
|
41
|
-
# get rid of bad descriptors
|
42
|
-
uri_socks.delete_if do |sock, uri|
|
43
|
-
begin
|
44
|
-
sock.recv_nonblock(1)
|
45
|
-
false # should never get here for HTTP, really...
|
46
|
-
rescue Errno::EAGAIN, Errno::EINTR
|
47
|
-
false
|
48
|
-
rescue
|
49
|
-
sock.close rescue nil
|
50
|
-
true
|
51
|
-
end
|
52
|
-
end
|
53
|
-
timeout -= (Time.now - t0)
|
54
|
-
retry if timeout >= 0
|
55
|
-
end
|
56
|
-
|
57
|
-
break unless r && r[1]
|
58
|
-
|
59
|
-
r[1].each do |sock|
|
60
|
-
begin
|
61
|
-
# we don't care about short/interrupted writes here, if the
|
62
|
-
# following request fails or blocks then the server is
|
63
|
-
# flat-out hopeless
|
64
|
-
sock.write_nonblock(
|
65
|
-
"HEAD #{uri_socks[sock].request_uri} HTTP/1.0\r\n\r\n")
|
66
|
-
sockets << sock
|
67
|
-
rescue
|
68
|
-
sock.close rescue nil
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
timeout -= (Time.now - t0)
|
73
|
-
end until (sockets[0] || timeout < 0)
|
74
|
-
|
75
|
-
[ sockets, timeout ]
|
76
|
-
end
|
77
|
-
|
78
|
-
# returns an array of URIs from uri_socks that are good
|
79
|
-
def get_readable_uris(sockets, uri_socks, expect, timeout)
|
80
|
-
ok_uris = []
|
81
|
-
|
82
|
-
begin
|
83
|
-
t0 = Time.now
|
84
|
-
r = IO.select(sockets, nil, nil, timeout > 0 ? timeout : 0) rescue nil
|
85
|
-
|
86
|
-
(r && r[0] ? r[0] : sockets).each do |sock|
|
87
|
-
buf = begin
|
88
|
-
sock.recv_nonblock(128, Socket::MSG_PEEK)
|
89
|
-
rescue Errno::EAGAIN, Errno::EINTR
|
90
|
-
next
|
91
|
-
rescue
|
92
|
-
sockets.delete(sock) # socket went bad
|
93
|
-
next
|
94
|
-
end
|
95
|
-
|
96
|
-
if buf && /\AHTTP\/[\d\.]+ #{expect} / =~ buf
|
97
|
-
ok_uris << uri_socks.delete(sock)
|
98
|
-
sock.close rescue nil
|
99
|
-
end
|
100
|
-
end
|
101
|
-
timeout -= (Time.now - t0)
|
102
|
-
end until ok_uris[0] || timeout < 0
|
103
|
-
|
104
|
-
ok_uris
|
105
|
-
end
|
106
|
-
|
107
|
-
end # module MogileFS::Network
|
data/test/test_network.rb
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
# -*- encoding: binary -*-
|
2
|
-
require 'test/setup'
|
3
|
-
require 'mogilefs'
|
4
|
-
require 'mogilefs/network'
|
5
|
-
|
6
|
-
class TestNetwork < Test::Unit::TestCase
|
7
|
-
include MogileFS::Network
|
8
|
-
|
9
|
-
def test_verify_uris
|
10
|
-
good = TempServer.new(Proc.new do |serv,port|
|
11
|
-
client,client_addr = serv.accept
|
12
|
-
client.readpartial(4096)
|
13
|
-
client.syswrite("HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n")
|
14
|
-
end)
|
15
|
-
bad = TempServer.new(Proc.new do |serv,port|
|
16
|
-
client, client_addr = serv.accept
|
17
|
-
client.close rescue nil
|
18
|
-
end)
|
19
|
-
|
20
|
-
good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
|
21
|
-
bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
|
22
|
-
ok = verify_uris([ good_uri, bad_uri ])
|
23
|
-
assert_equal [ good_uri ], ok
|
24
|
-
ensure
|
25
|
-
TempServer.destroy_all!
|
26
|
-
end
|
27
|
-
|
28
|
-
def test_verify_non_existent
|
29
|
-
good = TempServer.new(Proc.new do |serv,port|
|
30
|
-
client,client_addr = serv.accept
|
31
|
-
client.readpartial(4096)
|
32
|
-
client.syswrite("HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n")
|
33
|
-
end)
|
34
|
-
bad = TempServer.new(Proc.new { |serv,port| serv.close })
|
35
|
-
|
36
|
-
good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
|
37
|
-
bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
|
38
|
-
ok = verify_uris([ good_uri, bad_uri ])
|
39
|
-
assert_equal [ good_uri ], ok
|
40
|
-
ensure
|
41
|
-
TempServer.destroy_all!
|
42
|
-
end
|
43
|
-
|
44
|
-
def test_verify_all_bad
|
45
|
-
good = TempServer.new(Proc.new { |serv,port| serv.close })
|
46
|
-
bad = TempServer.new(Proc.new { |serv,port| serv.close })
|
47
|
-
|
48
|
-
good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
|
49
|
-
bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
|
50
|
-
ok = verify_uris([ good_uri, bad_uri ], '200', 1.0)
|
51
|
-
assert ok.empty?, "nothing returned"
|
52
|
-
ensure
|
53
|
-
TempServer.destroy_all!
|
54
|
-
end
|
55
|
-
|
56
|
-
end
|
data/test/test_util.rb
DELETED
@@ -1,121 +0,0 @@
|
|
1
|
-
# -*- encoding: binary -*-
|
2
|
-
require 'test/setup'
|
3
|
-
|
4
|
-
class TestMogileFS__Util < Test::Unit::TestCase
|
5
|
-
include MogileFS::Util
|
6
|
-
|
7
|
-
def test_mogilefs_write
|
8
|
-
rd, wr = IO.pipe
|
9
|
-
|
10
|
-
svr = Proc.new do |serv, port|
|
11
|
-
client, client_addr = serv.accept
|
12
|
-
client.sync = true
|
13
|
-
nr = 0
|
14
|
-
loop do
|
15
|
-
begin
|
16
|
-
nr += client.readpartial(16384).length
|
17
|
-
rescue EOFError
|
18
|
-
break
|
19
|
-
end
|
20
|
-
end
|
21
|
-
wr.syswrite("#{nr}\n")
|
22
|
-
client.close rescue nil
|
23
|
-
end
|
24
|
-
t = TempServer.new(svr)
|
25
|
-
s = Socket.mogilefs_new('127.0.0.1', t.port)
|
26
|
-
tmp = s.getsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF)
|
27
|
-
sndbuf_bytes = tmp.unpack('i')[0]
|
28
|
-
big_string = ' ' * (sndbuf_bytes * 10)
|
29
|
-
|
30
|
-
sent = s.send(big_string, 0)
|
31
|
-
assert(sent < big_string.length)
|
32
|
-
|
33
|
-
syswrite_full(s, big_string)
|
34
|
-
s.close rescue nil
|
35
|
-
IO.select([rd])
|
36
|
-
assert_equal((sent + big_string.length), rd.sysread(4096).to_i)
|
37
|
-
ensure
|
38
|
-
t.destroy!
|
39
|
-
end
|
40
|
-
|
41
|
-
def test_write_timeout
|
42
|
-
svr = Proc.new do |serv, port|
|
43
|
-
client, client_addr = serv.accept
|
44
|
-
client.sync = true
|
45
|
-
readed = client.readpartial(16384)
|
46
|
-
sleep
|
47
|
-
end
|
48
|
-
t = TempServer.new(svr)
|
49
|
-
s = Socket.mogilefs_new('127.0.0.1', t.port)
|
50
|
-
tmp = s.getsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF)
|
51
|
-
sndbuf_bytes = tmp.unpack('i')[0]
|
52
|
-
big_string = ' ' * (sndbuf_bytes * 10)
|
53
|
-
|
54
|
-
assert_raises(MogileFS::Timeout) { syswrite_full(s, big_string, 0.1) }
|
55
|
-
s.close rescue nil
|
56
|
-
ensure
|
57
|
-
t.destroy!
|
58
|
-
end
|
59
|
-
|
60
|
-
def test_sysread_slowly
|
61
|
-
nr = 10
|
62
|
-
str = 'abcde'
|
63
|
-
expect = str * nr
|
64
|
-
rd, wr = IO.pipe
|
65
|
-
pid = fork do
|
66
|
-
rd.close
|
67
|
-
nr.times do
|
68
|
-
syswrite_full(wr, str)
|
69
|
-
sleep(0.1)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
wr.close
|
73
|
-
buf = sysread_full(rd, expect.size)
|
74
|
-
assert_equal expect, buf
|
75
|
-
rd.close
|
76
|
-
ensure
|
77
|
-
Process.kill('TERM', pid) rescue nil
|
78
|
-
Process.waitpid(pid) rescue nil
|
79
|
-
end
|
80
|
-
|
81
|
-
def test_sysread_timeout
|
82
|
-
nr = 10
|
83
|
-
str = 'abcde'
|
84
|
-
expect = str * nr
|
85
|
-
rd, wr = IO.pipe
|
86
|
-
pid = fork do
|
87
|
-
rd.close
|
88
|
-
nr.times do
|
89
|
-
syswrite_full(wr, str)
|
90
|
-
sleep 1
|
91
|
-
end
|
92
|
-
end
|
93
|
-
wr.close
|
94
|
-
assert_raises(MogileFS::Timeout) { sysread_full(rd, expect.size, 0.1) }
|
95
|
-
rd.close
|
96
|
-
ensure
|
97
|
-
Process.kill('TERM', pid) rescue nil
|
98
|
-
Process.waitpid(pid) rescue nil
|
99
|
-
end
|
100
|
-
|
101
|
-
def test_sysread_full_timeout
|
102
|
-
nr = 100
|
103
|
-
str = 'abcde'
|
104
|
-
expect = str * nr
|
105
|
-
rd, wr = IO.pipe
|
106
|
-
pid = fork do
|
107
|
-
rd.close
|
108
|
-
nr.times do
|
109
|
-
syswrite_full(wr, str)
|
110
|
-
sleep 0.01
|
111
|
-
end
|
112
|
-
end
|
113
|
-
wr.close
|
114
|
-
assert_raises(MogileFS::Timeout) { sysread_full(rd,expect.size,0.1,true) }
|
115
|
-
rd.close
|
116
|
-
ensure
|
117
|
-
Process.kill('TERM', pid) rescue nil
|
118
|
-
Process.waitpid(pid) rescue nil
|
119
|
-
end
|
120
|
-
|
121
|
-
end
|