mogilefs-client 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +3 -0
- data/.gitignore +1 -0
- data/GIT-VERSION-GEN +1 -1
- data/GNUmakefile +2 -10
- data/History +3 -0
- data/Rakefile +8 -0
- data/bin/mog +18 -38
- data/examples/stale_fid_checker.rb +108 -0
- data/lib/mogilefs.rb +3 -1
- data/lib/mogilefs/admin.rb +2 -1
- data/lib/mogilefs/backend.rb +16 -44
- data/lib/mogilefs/chunker.rb +11 -5
- data/lib/mogilefs/client.rb +6 -1
- data/lib/mogilefs/http_file.rb +51 -88
- data/lib/mogilefs/mogilefs.rb +91 -39
- data/lib/mogilefs/new_file.rb +78 -0
- data/lib/mogilefs/new_file/common.rb +89 -0
- data/lib/mogilefs/new_file/content_range.rb +105 -0
- data/lib/mogilefs/new_file/stream.rb +91 -0
- data/lib/mogilefs/new_file/tempfile.rb +24 -0
- data/lib/mogilefs/new_file/writer.rb +57 -0
- data/lib/mogilefs/pool.rb +18 -10
- data/lib/mogilefs/socket/kgio.rb +4 -3
- data/lib/mogilefs/socket/pure_ruby.rb +3 -3
- data/lib/mogilefs/socket_common.rb +7 -6
- data/test/fresh.rb +5 -2
- data/test/test_backend.rb +16 -8
- data/test/test_mogilefs_integration.rb +121 -0
- data/test/test_mogstored_rack.rb +112 -6
- data/test/test_pool.rb +26 -0
- metadata +13 -10
- data/Manifest.txt +0 -60
- data/examples/mogstored_rack.rb +0 -188
- data/test/test_unit_mogstored_rack.rb +0 -72
@@ -0,0 +1,89 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# here are internal implementation details, do not use them in your code
|
3
|
+
require 'socket'
|
4
|
+
require 'uri'
|
5
|
+
require 'digest/md5'
|
6
|
+
require 'mogilefs/chunker'
|
7
|
+
|
8
|
+
module MogileFS::NewFile::Common
|
9
|
+
# :stopdoc:
|
10
|
+
class RetryableError < MogileFS::Error; end
|
11
|
+
class EmptyResponseError < RetryableError; end
|
12
|
+
class BadResponseError < RetryableError; end
|
13
|
+
class UnparseableResponseError < RetryableError; end
|
14
|
+
class NoStorageNodesError < MogileFS::Error
|
15
|
+
def message; 'Unable to open socket to storage node'; end
|
16
|
+
end
|
17
|
+
class NonRetryableError < MogileFS::Error; end
|
18
|
+
|
19
|
+
MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
|
20
|
+
|
21
|
+
def read_response(sock)
|
22
|
+
tout = @opts[:new_file_max_time] || 3600.0
|
23
|
+
start_time = @opts[:start_time] and tout -= Time.now - start_time
|
24
|
+
case line = sock.timed_read(23, "", tout > 0.0 ? tout : 0)
|
25
|
+
when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
|
26
|
+
when nil
|
27
|
+
raise EmptyResponseError, 'Unable to read response line from server'
|
28
|
+
when %r{^HTTP/\d\.\d\s+(\d+)}
|
29
|
+
raise BadResponseError, "HTTP response status from upload: #$1"
|
30
|
+
else
|
31
|
+
raise UnparseableResponseError,
|
32
|
+
"Response line not understood: #{line.inspect}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def create_close(devid, uri, bytes_uploaded)
|
37
|
+
args = {
|
38
|
+
:fid => @opts[:fid],
|
39
|
+
:devid => devid,
|
40
|
+
:key => @opts[:key],
|
41
|
+
:domain => @opts[:domain],
|
42
|
+
:size => bytes_uploaded,
|
43
|
+
:path => uri.to_s,
|
44
|
+
}
|
45
|
+
if @md5
|
46
|
+
args[:checksum] = "MD5:#{@md5.hexdigest}"
|
47
|
+
elsif String === @opts[:content_md5]
|
48
|
+
hex = @opts[:content_md5].unpack('m')[0].unpack('H*')[0]
|
49
|
+
args[:checksum] = "MD5:#{hex}"
|
50
|
+
end
|
51
|
+
args[:checksumverify] = 1 if @opts[:checksumverify]
|
52
|
+
backend = @opts[:backend]
|
53
|
+
|
54
|
+
# upload could've taken a long time, ping and try to ensure socket
|
55
|
+
# is valid to minimize (but not completely eliminate) the chance
|
56
|
+
# create_close hits a stale socket (while reading the response after
|
57
|
+
# writing to it) and becomes non-retryable. We treat create_close
|
58
|
+
# specially as its less idempotent than any other command
|
59
|
+
# (even other non-idempotent ones). There may be no hope of retrying
|
60
|
+
# the upload at all if data was streamed and calling create_close
|
61
|
+
# twice will hurt us...
|
62
|
+
backend.noop
|
63
|
+
|
64
|
+
backend.create_close(args)
|
65
|
+
bytes_uploaded
|
66
|
+
end
|
67
|
+
|
68
|
+
# aggressive keepalive settings on Linux + Ruby 1.9.2+
|
69
|
+
TCP_KEEPALIVE = {
|
70
|
+
:TCP_KEEPIDLE => 60, # seconds time before keepalive packet is sent
|
71
|
+
:TCP_KEEPINTVL => 5,
|
72
|
+
:TCP_KEEPCNT => 2, # number of retries
|
73
|
+
}
|
74
|
+
|
75
|
+
req_consts = TCP_KEEPALIVE.keys
|
76
|
+
if (Socket.constants & req_consts).size == req_consts.size
|
77
|
+
def set_socket_options(sock)
|
78
|
+
sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1)
|
79
|
+
TCP_KEEPALIVE.each do |k,v|
|
80
|
+
sock.setsockopt(:IPPROTO_TCP, k, v)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
else
|
84
|
+
def set_socket_options(sock)
|
85
|
+
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
# :startdoc:
|
89
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# here are internal implementation details, do not rely on them in your code
|
3
|
+
require 'net/http'
|
4
|
+
require 'mogilefs/new_file/writer'
|
5
|
+
|
6
|
+
# an IO-like object
|
7
|
+
class MogileFS::NewFile::ContentRange
|
8
|
+
include MogileFS::NewFile::Writer
|
9
|
+
include MogileFS::NewFile::Common
|
10
|
+
attr_reader :md5
|
11
|
+
|
12
|
+
# :stopdoc:
|
13
|
+
begin
|
14
|
+
require 'net/http/persistent'
|
15
|
+
NHP = Net::HTTP::Persistent.new('mogilefs')
|
16
|
+
|
17
|
+
def hit(uri, req)
|
18
|
+
NHP.request(uri, req).value
|
19
|
+
end
|
20
|
+
rescue LoadError
|
21
|
+
def hit(uri, req)
|
22
|
+
Net::HTTP.start(uri.host, uri.port) { |h| h.request(req).value }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
# :startdoc:
|
26
|
+
|
27
|
+
def initialize(dests, opts) # :nodoc:
|
28
|
+
@dests = dests
|
29
|
+
@opts = opts
|
30
|
+
@devid = @uri = @md5 = nil
|
31
|
+
@bytes_uploaded = 0
|
32
|
+
@errors = []
|
33
|
+
end
|
34
|
+
|
35
|
+
def get_dest # :nodoc:
|
36
|
+
return [ @devid, @uri ] if @uri
|
37
|
+
rv = @dests.shift or no_nodes!
|
38
|
+
rv[1] = URI.parse(rv[1])
|
39
|
+
rv
|
40
|
+
end
|
41
|
+
|
42
|
+
def no_nodes! # :nodoc:
|
43
|
+
raise NoStorageNodesError,
|
44
|
+
"all paths failed with PUT: #{@errors.join(', ')}", []
|
45
|
+
end
|
46
|
+
|
47
|
+
def request_for(uri, buf) # :nodoc:
|
48
|
+
put = Net::HTTP::Put.new(uri.path)
|
49
|
+
put["Content-Type"] = "application/octet-stream"
|
50
|
+
put["Content-MD5"] = [ Digest::MD5.digest(buf) ].pack("m").chomp!
|
51
|
+
if @bytes_uploaded > 0
|
52
|
+
last_byte = @bytes_uploaded + buf.bytesize - 1
|
53
|
+
put["Content-Range"] = "bytes #@bytes_uploaded-#{last_byte}/*"
|
54
|
+
end
|
55
|
+
put.body = buf
|
56
|
+
|
57
|
+
put
|
58
|
+
end
|
59
|
+
|
60
|
+
# see IO#write
|
61
|
+
def write(buf)
|
62
|
+
buf = String buf
|
63
|
+
len = buf.bytesize
|
64
|
+
return 0 if 0 == len
|
65
|
+
|
66
|
+
devid, uri = get_dest
|
67
|
+
put = request_for(uri, buf)
|
68
|
+
begin
|
69
|
+
hit(uri, put) # raises on error
|
70
|
+
rescue => e
|
71
|
+
raise if @bytes_uploaded > 0
|
72
|
+
|
73
|
+
# nothing uploaded, try another dest
|
74
|
+
@errors << "#{uri.to_s} - #{e.message} (#{e.class})"
|
75
|
+
devid, uri = get_dest
|
76
|
+
put = request_for(uri, buf)
|
77
|
+
retry
|
78
|
+
end
|
79
|
+
|
80
|
+
@uri, @devid = uri, devid if 0 == @bytes_uploaded
|
81
|
+
@bytes_uploaded += len
|
82
|
+
len
|
83
|
+
end
|
84
|
+
|
85
|
+
# called on close, do not use
|
86
|
+
def commit # :nodoc:
|
87
|
+
zero_byte_special if @bytes_uploaded == 0
|
88
|
+
|
89
|
+
create_close(@devid, @uri, @bytes_uploaded)
|
90
|
+
end
|
91
|
+
|
92
|
+
# special case for zero-byte files :<
|
93
|
+
def zero_byte_special # :nodoc:
|
94
|
+
@devid, @uri = get_dest
|
95
|
+
put = request_for(@uri, "")
|
96
|
+
begin
|
97
|
+
hit(@uri, put) # raises on error
|
98
|
+
rescue => e
|
99
|
+
@errors << "#{@uri.to_s} - #{e.message} (#{e.class})"
|
100
|
+
@devid, @uri = get_dest
|
101
|
+
put = request_for(@uri, "")
|
102
|
+
retry
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# here are internal implementation details, do not use them in your code
|
3
|
+
|
4
|
+
require 'mogilefs/new_file/writer'
|
5
|
+
|
6
|
+
class MogileFS::NewFile::Stream
|
7
|
+
attr_reader :to_io
|
8
|
+
attr_reader :md5
|
9
|
+
|
10
|
+
include MogileFS::NewFile::Writer
|
11
|
+
include MogileFS::NewFile::Common
|
12
|
+
|
13
|
+
def initialize(dests, opts)
|
14
|
+
@opts = opts
|
15
|
+
@md5 = nil
|
16
|
+
@bytes_uploaded = 0
|
17
|
+
dests.each do |devid, path|
|
18
|
+
begin
|
19
|
+
uri = URI.parse(path)
|
20
|
+
sock = MogileFS::Socket.tcp(uri.host, uri.port)
|
21
|
+
set_socket_options(sock)
|
22
|
+
start_sock(sock, uri) # raise on errors
|
23
|
+
@to_io = sock
|
24
|
+
@uri = uri
|
25
|
+
@devid = devid
|
26
|
+
if ! @md5 && @opts[:content_length]
|
27
|
+
@writer = @to_io
|
28
|
+
else
|
29
|
+
@writer = MogileFS::Chunker.new(@to_io, @md5, @opts[:content_md5])
|
30
|
+
end
|
31
|
+
return
|
32
|
+
rescue SystemCallError => e
|
33
|
+
sock.close if sock && ! sock.closed?
|
34
|
+
errors ||= []
|
35
|
+
errors << "#{path} - #{e.message} (#{e.class})"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
raise NoStorageNodesError,
|
40
|
+
"all paths failed with PUT: #{errors.join(', ')}", []
|
41
|
+
end
|
42
|
+
|
43
|
+
def write(buf)
|
44
|
+
buf = String buf
|
45
|
+
return 0 if 0 == buf.size
|
46
|
+
rv = @writer.write(buf)
|
47
|
+
@bytes_uploaded += rv
|
48
|
+
rv
|
49
|
+
end
|
50
|
+
|
51
|
+
def commit
|
52
|
+
@writer.flush
|
53
|
+
|
54
|
+
clen = @opts[:content_length]
|
55
|
+
if clen && @bytes_uploaded != clen
|
56
|
+
raise MogileFS::SizeMismatchError,
|
57
|
+
"did not upload expected content_length: #{clen} uploaded: " \
|
58
|
+
"#@bytes_uploaded"
|
59
|
+
end
|
60
|
+
read_response(@to_io) # raises on errors
|
61
|
+
create_close(@devid, @uri, @bytes_uploaded)
|
62
|
+
ensure
|
63
|
+
@to_io.close if @to_io && ! @to_io.closed?
|
64
|
+
end
|
65
|
+
|
66
|
+
def start_sock(sock, uri)
|
67
|
+
host_with_port = "#{uri.host}:#{uri.port}"
|
68
|
+
headers = "PUT #{uri.request_uri} HTTP/1.1\r\n" \
|
69
|
+
"Host: #{host_with_port}\r\n" \
|
70
|
+
|
71
|
+
content_md5 = @opts[:content_md5]
|
72
|
+
if String === content_md5
|
73
|
+
headers << "Content-MD5: #{content_md5}\r\n"
|
74
|
+
elsif content_md5.respond_to?(:call) ||
|
75
|
+
:trailer == content_md5 ||
|
76
|
+
MD5_TRAILER_NODES[host_with_port]
|
77
|
+
@md5 = Digest::MD5.new
|
78
|
+
headers << "Trailer: Content-MD5\r\n"
|
79
|
+
end
|
80
|
+
|
81
|
+
if ! @md5 && clen = @opts[:content_length]
|
82
|
+
headers << "Content-Length: #{clen}\r\n"
|
83
|
+
else
|
84
|
+
headers << "Transfer-Encoding: chunked\r\n"
|
85
|
+
end
|
86
|
+
|
87
|
+
sock.write(headers << "\r\n")
|
88
|
+
end
|
89
|
+
|
90
|
+
alias syswrite write
|
91
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# here are internal implementation details, do not rely on them in your code
|
3
|
+
require 'tempfile'
|
4
|
+
require 'mogilefs/http_file'
|
5
|
+
|
6
|
+
class MogileFS::NewFile::Tempfile < Tempfile
|
7
|
+
def initialize(*args)
|
8
|
+
@mogilefs_httpfile_args = args
|
9
|
+
super("mogilefs-client")
|
10
|
+
unlink
|
11
|
+
end
|
12
|
+
|
13
|
+
def commit
|
14
|
+
rewind
|
15
|
+
tmp = MogileFS::HTTPFile.new(*@mogilefs_httpfile_args)
|
16
|
+
tmp.big_io = to_io
|
17
|
+
tmp.commit
|
18
|
+
end
|
19
|
+
|
20
|
+
def close
|
21
|
+
commit
|
22
|
+
super
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
#
|
3
|
+
# All objects yielded or returned by MogileFS::MogileFS#new_file should
|
4
|
+
# conform to this interface (based on existing IO methods). These objects
|
5
|
+
# should be considered write-only.
|
6
|
+
module MogileFS::NewFile::Writer
|
7
|
+
|
8
|
+
# see IO#puts
|
9
|
+
def puts(*args)
|
10
|
+
args.each do |obj|
|
11
|
+
write(obj)
|
12
|
+
write("\n")
|
13
|
+
end
|
14
|
+
nil
|
15
|
+
end
|
16
|
+
|
17
|
+
# see IO#putc
|
18
|
+
def putc(ch)
|
19
|
+
write(ch.respond_to?(:chr) ? ch.chr : ch[0])
|
20
|
+
ch
|
21
|
+
end
|
22
|
+
|
23
|
+
# see IO#print
|
24
|
+
def print(*args)
|
25
|
+
args = [ $_ ] unless args[0]
|
26
|
+
write(args.shift)
|
27
|
+
args.each do |obj|
|
28
|
+
write(obj)
|
29
|
+
write($,) if $,
|
30
|
+
end
|
31
|
+
write($\) if $\
|
32
|
+
nil
|
33
|
+
end
|
34
|
+
|
35
|
+
# see IO#printf
|
36
|
+
def printf(*args)
|
37
|
+
write(sprintf(*args))
|
38
|
+
nil
|
39
|
+
end
|
40
|
+
|
41
|
+
# see IO#<<
|
42
|
+
def <<(str)
|
43
|
+
write(str)
|
44
|
+
self
|
45
|
+
end
|
46
|
+
|
47
|
+
# This will issue the +create_close+ command to the MogileFS tracker
|
48
|
+
# and finalize the creation of a new file. This returns +nil+ on
|
49
|
+
# success and will raise IOError if called twice. For non-streaming
|
50
|
+
# implementations, this will initiate and finalize the upload.
|
51
|
+
#
|
52
|
+
# see IO#close
|
53
|
+
def close
|
54
|
+
commit
|
55
|
+
nil
|
56
|
+
end
|
57
|
+
end
|
data/lib/mogilefs/pool.rb
CHANGED
@@ -3,23 +3,30 @@ require 'thread'
|
|
3
3
|
|
4
4
|
class MogileFS::Pool
|
5
5
|
|
6
|
+
# Must be a positive Integer that is greater than :purge_keep
|
7
|
+
# Default: 5
|
8
|
+
attr_accessor :purge_threshold
|
9
|
+
|
10
|
+
# Must be a positive Integer that is smaller than :purge_threshold
|
11
|
+
# Default: 2
|
12
|
+
attr_accessor :purge_keep
|
13
|
+
|
6
14
|
class BadObjectError < RuntimeError; end
|
7
15
|
|
8
16
|
def initialize(klass, *args)
|
9
17
|
@args = args
|
10
18
|
@klass = klass
|
11
19
|
@queue = Queue.new
|
12
|
-
@objects =
|
20
|
+
@objects = {}
|
21
|
+
@purge_threshold = 5
|
22
|
+
@purge_keep = 2
|
13
23
|
end
|
14
24
|
|
15
25
|
def get
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
@objects << object
|
21
|
-
end
|
22
|
-
object
|
26
|
+
@queue.pop true
|
27
|
+
rescue ThreadError
|
28
|
+
object = @klass.new(*@args)
|
29
|
+
@objects[object] = object
|
23
30
|
end
|
24
31
|
|
25
32
|
def put(o)
|
@@ -38,11 +45,12 @@ class MogileFS::Pool
|
|
38
45
|
end
|
39
46
|
|
40
47
|
def purge
|
41
|
-
return if @queue.length <
|
48
|
+
return if @queue.length < @purge_threshold
|
42
49
|
begin
|
43
|
-
until @queue.length <=
|
50
|
+
until @queue.length <= @purge_keep
|
44
51
|
obj = @queue.pop true
|
45
52
|
@objects.delete obj
|
53
|
+
obj.backend.shutdown if MogileFS::Client === obj
|
46
54
|
end
|
47
55
|
rescue ThreadError
|
48
56
|
end
|
data/lib/mogilefs/socket/kgio.rb
CHANGED
@@ -22,7 +22,7 @@ class MogileFS::Socket < Kgio::Socket
|
|
22
22
|
def timed_read(len, dst = "", timeout = 5)
|
23
23
|
case rc = kgio_tryread(len, dst)
|
24
24
|
when :wait_readable
|
25
|
-
kgio_wait_readable(timeout) or unreadable_socket!
|
25
|
+
kgio_wait_readable(timeout) or unreadable_socket!(timeout)
|
26
26
|
else
|
27
27
|
return rc
|
28
28
|
end while true
|
@@ -31,7 +31,7 @@ class MogileFS::Socket < Kgio::Socket
|
|
31
31
|
def timed_peek(len, dst, timeout = 5)
|
32
32
|
case rc = kgio_trypeek(len, dst)
|
33
33
|
when :wait_readable
|
34
|
-
kgio_wait_readable(timeout) or unreadable_socket!
|
34
|
+
kgio_wait_readable(timeout) or unreadable_socket!(timeout)
|
35
35
|
else
|
36
36
|
return rc
|
37
37
|
end while true
|
@@ -42,7 +42,8 @@ class MogileFS::Socket < Kgio::Socket
|
|
42
42
|
expect = buf.bytesize
|
43
43
|
case rc = kgio_trywrite(buf)
|
44
44
|
when :wait_writable
|
45
|
-
kgio_wait_writable(timeout) or
|
45
|
+
kgio_wait_writable(timeout) or
|
46
|
+
request_truncated!(written, expect, timeout)
|
46
47
|
when String
|
47
48
|
written += expect - rc.bytesize
|
48
49
|
buf = rc
|