mogilefs-client 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,89 @@
1
+ # -*- encoding: binary -*-
2
+ # here are internal implementation details, do not use them in your code
3
+ require 'socket'
4
+ require 'uri'
5
+ require 'digest/md5'
6
+ require 'mogilefs/chunker'
7
+
8
+ module MogileFS::NewFile::Common
9
+ # :stopdoc:
10
+ class RetryableError < MogileFS::Error; end
11
+ class EmptyResponseError < RetryableError; end
12
+ class BadResponseError < RetryableError; end
13
+ class UnparseableResponseError < RetryableError; end
14
+ class NoStorageNodesError < MogileFS::Error
15
+ def message; 'Unable to open socket to storage node'; end
16
+ end
17
+ class NonRetryableError < MogileFS::Error; end
18
+
19
+ MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
20
+
21
+ def read_response(sock)
22
+ tout = @opts[:new_file_max_time] || 3600.0
23
+ start_time = @opts[:start_time] and tout -= Time.now - start_time
24
+ case line = sock.timed_read(23, "", tout > 0.0 ? tout : 0)
25
+ when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
26
+ when nil
27
+ raise EmptyResponseError, 'Unable to read response line from server'
28
+ when %r{^HTTP/\d\.\d\s+(\d+)}
29
+ raise BadResponseError, "HTTP response status from upload: #$1"
30
+ else
31
+ raise UnparseableResponseError,
32
+ "Response line not understood: #{line.inspect}"
33
+ end
34
+ end
35
+
36
+ def create_close(devid, uri, bytes_uploaded)
37
+ args = {
38
+ :fid => @opts[:fid],
39
+ :devid => devid,
40
+ :key => @opts[:key],
41
+ :domain => @opts[:domain],
42
+ :size => bytes_uploaded,
43
+ :path => uri.to_s,
44
+ }
45
+ if @md5
46
+ args[:checksum] = "MD5:#{@md5.hexdigest}"
47
+ elsif String === @opts[:content_md5]
48
+ hex = @opts[:content_md5].unpack('m')[0].unpack('H*')[0]
49
+ args[:checksum] = "MD5:#{hex}"
50
+ end
51
+ args[:checksumverify] = 1 if @opts[:checksumverify]
52
+ backend = @opts[:backend]
53
+
54
+ # upload could've taken a long time, ping and try to ensure socket
55
+ # is valid to minimize (but not completely eliminate) the chance
56
+ # create_close hits a stale socket (while reading the response after
57
+ # writing to it) and becomes non-retryable. We treat create_close
58
+ # specially as its less idempotent than any other command
59
+ # (even other non-idempotent ones). There may be no hope of retrying
60
+ # the upload at all if data was streamed and calling create_close
61
+ # twice will hurt us...
62
+ backend.noop
63
+
64
+ backend.create_close(args)
65
+ bytes_uploaded
66
+ end
67
+
68
+ # aggressive keepalive settings on Linux + Ruby 1.9.2+
69
+ TCP_KEEPALIVE = {
70
+ :TCP_KEEPIDLE => 60, # seconds time before keepalive packet is sent
71
+ :TCP_KEEPINTVL => 5,
72
+ :TCP_KEEPCNT => 2, # number of retries
73
+ }
74
+
75
+ req_consts = TCP_KEEPALIVE.keys
76
+ if (Socket.constants & req_consts).size == req_consts.size
77
+ def set_socket_options(sock)
78
+ sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1)
79
+ TCP_KEEPALIVE.each do |k,v|
80
+ sock.setsockopt(:IPPROTO_TCP, k, v)
81
+ end
82
+ end
83
+ else
84
+ def set_socket_options(sock)
85
+ sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1)
86
+ end
87
+ end
88
+ # :startdoc:
89
+ end
@@ -0,0 +1,105 @@
1
+ # -*- encoding: binary -*-
2
+ # here are internal implementation details, do not rely on them in your code
3
+ require 'net/http'
4
+ require 'mogilefs/new_file/writer'
5
+
6
+ # an IO-like object
7
+ class MogileFS::NewFile::ContentRange
8
+ include MogileFS::NewFile::Writer
9
+ include MogileFS::NewFile::Common
10
+ attr_reader :md5
11
+
12
+ # :stopdoc:
13
+ begin
14
+ require 'net/http/persistent'
15
+ NHP = Net::HTTP::Persistent.new('mogilefs')
16
+
17
+ def hit(uri, req)
18
+ NHP.request(uri, req).value
19
+ end
20
+ rescue LoadError
21
+ def hit(uri, req)
22
+ Net::HTTP.start(uri.host, uri.port) { |h| h.request(req).value }
23
+ end
24
+ end
25
+ # :startdoc:
26
+
27
+ def initialize(dests, opts) # :nodoc:
28
+ @dests = dests
29
+ @opts = opts
30
+ @devid = @uri = @md5 = nil
31
+ @bytes_uploaded = 0
32
+ @errors = []
33
+ end
34
+
35
+ def get_dest # :nodoc:
36
+ return [ @devid, @uri ] if @uri
37
+ rv = @dests.shift or no_nodes!
38
+ rv[1] = URI.parse(rv[1])
39
+ rv
40
+ end
41
+
42
+ def no_nodes! # :nodoc:
43
+ raise NoStorageNodesError,
44
+ "all paths failed with PUT: #{@errors.join(', ')}", []
45
+ end
46
+
47
+ def request_for(uri, buf) # :nodoc:
48
+ put = Net::HTTP::Put.new(uri.path)
49
+ put["Content-Type"] = "application/octet-stream"
50
+ put["Content-MD5"] = [ Digest::MD5.digest(buf) ].pack("m").chomp!
51
+ if @bytes_uploaded > 0
52
+ last_byte = @bytes_uploaded + buf.bytesize - 1
53
+ put["Content-Range"] = "bytes #@bytes_uploaded-#{last_byte}/*"
54
+ end
55
+ put.body = buf
56
+
57
+ put
58
+ end
59
+
60
+ # see IO#write
61
+ def write(buf)
62
+ buf = String buf
63
+ len = buf.bytesize
64
+ return 0 if 0 == len
65
+
66
+ devid, uri = get_dest
67
+ put = request_for(uri, buf)
68
+ begin
69
+ hit(uri, put) # raises on error
70
+ rescue => e
71
+ raise if @bytes_uploaded > 0
72
+
73
+ # nothing uploaded, try another dest
74
+ @errors << "#{uri.to_s} - #{e.message} (#{e.class})"
75
+ devid, uri = get_dest
76
+ put = request_for(uri, buf)
77
+ retry
78
+ end
79
+
80
+ @uri, @devid = uri, devid if 0 == @bytes_uploaded
81
+ @bytes_uploaded += len
82
+ len
83
+ end
84
+
85
+ # called on close, do not use
86
+ def commit # :nodoc:
87
+ zero_byte_special if @bytes_uploaded == 0
88
+
89
+ create_close(@devid, @uri, @bytes_uploaded)
90
+ end
91
+
92
+ # special case for zero-byte files :<
93
+ def zero_byte_special # :nodoc:
94
+ @devid, @uri = get_dest
95
+ put = request_for(@uri, "")
96
+ begin
97
+ hit(@uri, put) # raises on error
98
+ rescue => e
99
+ @errors << "#{@uri.to_s} - #{e.message} (#{e.class})"
100
+ @devid, @uri = get_dest
101
+ put = request_for(@uri, "")
102
+ retry
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,91 @@
1
+ # -*- encoding: binary -*-
2
+ # here are internal implementation details, do not use them in your code
3
+
4
+ require 'mogilefs/new_file/writer'
5
+
6
+ class MogileFS::NewFile::Stream
7
+ attr_reader :to_io
8
+ attr_reader :md5
9
+
10
+ include MogileFS::NewFile::Writer
11
+ include MogileFS::NewFile::Common
12
+
13
+ def initialize(dests, opts)
14
+ @opts = opts
15
+ @md5 = nil
16
+ @bytes_uploaded = 0
17
+ dests.each do |devid, path|
18
+ begin
19
+ uri = URI.parse(path)
20
+ sock = MogileFS::Socket.tcp(uri.host, uri.port)
21
+ set_socket_options(sock)
22
+ start_sock(sock, uri) # raise on errors
23
+ @to_io = sock
24
+ @uri = uri
25
+ @devid = devid
26
+ if ! @md5 && @opts[:content_length]
27
+ @writer = @to_io
28
+ else
29
+ @writer = MogileFS::Chunker.new(@to_io, @md5, @opts[:content_md5])
30
+ end
31
+ return
32
+ rescue SystemCallError => e
33
+ sock.close if sock && ! sock.closed?
34
+ errors ||= []
35
+ errors << "#{path} - #{e.message} (#{e.class})"
36
+ end
37
+ end
38
+
39
+ raise NoStorageNodesError,
40
+ "all paths failed with PUT: #{errors.join(', ')}", []
41
+ end
42
+
43
+ def write(buf)
44
+ buf = String buf
45
+ return 0 if 0 == buf.size
46
+ rv = @writer.write(buf)
47
+ @bytes_uploaded += rv
48
+ rv
49
+ end
50
+
51
+ def commit
52
+ @writer.flush
53
+
54
+ clen = @opts[:content_length]
55
+ if clen && @bytes_uploaded != clen
56
+ raise MogileFS::SizeMismatchError,
57
+ "did not upload expected content_length: #{clen} uploaded: " \
58
+ "#@bytes_uploaded"
59
+ end
60
+ read_response(@to_io) # raises on errors
61
+ create_close(@devid, @uri, @bytes_uploaded)
62
+ ensure
63
+ @to_io.close if @to_io && ! @to_io.closed?
64
+ end
65
+
66
+ def start_sock(sock, uri)
67
+ host_with_port = "#{uri.host}:#{uri.port}"
68
+ headers = "PUT #{uri.request_uri} HTTP/1.1\r\n" \
69
+ "Host: #{host_with_port}\r\n" \
70
+
71
+ content_md5 = @opts[:content_md5]
72
+ if String === content_md5
73
+ headers << "Content-MD5: #{content_md5}\r\n"
74
+ elsif content_md5.respond_to?(:call) ||
75
+ :trailer == content_md5 ||
76
+ MD5_TRAILER_NODES[host_with_port]
77
+ @md5 = Digest::MD5.new
78
+ headers << "Trailer: Content-MD5\r\n"
79
+ end
80
+
81
+ if ! @md5 && clen = @opts[:content_length]
82
+ headers << "Content-Length: #{clen}\r\n"
83
+ else
84
+ headers << "Transfer-Encoding: chunked\r\n"
85
+ end
86
+
87
+ sock.write(headers << "\r\n")
88
+ end
89
+
90
+ alias syswrite write
91
+ end
@@ -0,0 +1,24 @@
1
+ # -*- encoding: binary -*-
2
+ # here are internal implementation details, do not rely on them in your code
3
+ require 'tempfile'
4
+ require 'mogilefs/http_file'
5
+
6
+ class MogileFS::NewFile::Tempfile < Tempfile
7
+ def initialize(*args)
8
+ @mogilefs_httpfile_args = args
9
+ super("mogilefs-client")
10
+ unlink
11
+ end
12
+
13
+ def commit
14
+ rewind
15
+ tmp = MogileFS::HTTPFile.new(*@mogilefs_httpfile_args)
16
+ tmp.big_io = to_io
17
+ tmp.commit
18
+ end
19
+
20
+ def close
21
+ commit
22
+ super
23
+ end
24
+ end
@@ -0,0 +1,57 @@
1
+ # -*- encoding: binary -*-
2
+ #
3
+ # All objects yielded or returned by MogileFS::MogileFS#new_file should
4
+ # conform to this interface (based on existing IO methods). These objects
5
+ # should be considered write-only.
6
+ module MogileFS::NewFile::Writer
7
+
8
+ # see IO#puts
9
+ def puts(*args)
10
+ args.each do |obj|
11
+ write(obj)
12
+ write("\n")
13
+ end
14
+ nil
15
+ end
16
+
17
+ # see IO#putc
18
+ def putc(ch)
19
+ write(ch.respond_to?(:chr) ? ch.chr : ch[0])
20
+ ch
21
+ end
22
+
23
+ # see IO#print
24
+ def print(*args)
25
+ args = [ $_ ] unless args[0]
26
+ write(args.shift)
27
+ args.each do |obj|
28
+ write(obj)
29
+ write($,) if $,
30
+ end
31
+ write($\) if $\
32
+ nil
33
+ end
34
+
35
+ # see IO#printf
36
+ def printf(*args)
37
+ write(sprintf(*args))
38
+ nil
39
+ end
40
+
41
+ # see IO#<<
42
+ def <<(str)
43
+ write(str)
44
+ self
45
+ end
46
+
47
+ # This will issue the +create_close+ command to the MogileFS tracker
48
+ # and finalize the creation of a new file. This returns +nil+ on
49
+ # success and will raise IOError if called twice. For non-streaming
50
+ # implementations, this will initiate and finalize the upload.
51
+ #
52
+ # see IO#close
53
+ def close
54
+ commit
55
+ nil
56
+ end
57
+ end
data/lib/mogilefs/pool.rb CHANGED
@@ -3,23 +3,30 @@ require 'thread'
3
3
 
4
4
  class MogileFS::Pool
5
5
 
6
+ # Must be a positive Integer that is greater than :purge_keep
7
+ # Default: 5
8
+ attr_accessor :purge_threshold
9
+
10
+ # Must be a positive Integer that is smaller than :purge_threshold
11
+ # Default: 2
12
+ attr_accessor :purge_keep
13
+
6
14
  class BadObjectError < RuntimeError; end
7
15
 
8
16
  def initialize(klass, *args)
9
17
  @args = args
10
18
  @klass = klass
11
19
  @queue = Queue.new
12
- @objects = []
20
+ @objects = {}
21
+ @purge_threshold = 5
22
+ @purge_keep = 2
13
23
  end
14
24
 
15
25
  def get
16
- begin
17
- object = @queue.pop true
18
- rescue ThreadError
19
- object = @klass.new(*@args)
20
- @objects << object
21
- end
22
- object
26
+ @queue.pop true
27
+ rescue ThreadError
28
+ object = @klass.new(*@args)
29
+ @objects[object] = object
23
30
  end
24
31
 
25
32
  def put(o)
@@ -38,11 +45,12 @@ class MogileFS::Pool
38
45
  end
39
46
 
40
47
  def purge
41
- return if @queue.length < 5
48
+ return if @queue.length < @purge_threshold
42
49
  begin
43
- until @queue.length <= 2 do
50
+ until @queue.length <= @purge_keep
44
51
  obj = @queue.pop true
45
52
  @objects.delete obj
53
+ obj.backend.shutdown if MogileFS::Client === obj
46
54
  end
47
55
  rescue ThreadError
48
56
  end
@@ -22,7 +22,7 @@ class MogileFS::Socket < Kgio::Socket
22
22
  def timed_read(len, dst = "", timeout = 5)
23
23
  case rc = kgio_tryread(len, dst)
24
24
  when :wait_readable
25
- kgio_wait_readable(timeout) or unreadable_socket!
25
+ kgio_wait_readable(timeout) or unreadable_socket!(timeout)
26
26
  else
27
27
  return rc
28
28
  end while true
@@ -31,7 +31,7 @@ class MogileFS::Socket < Kgio::Socket
31
31
  def timed_peek(len, dst, timeout = 5)
32
32
  case rc = kgio_trypeek(len, dst)
33
33
  when :wait_readable
34
- kgio_wait_readable(timeout) or unreadable_socket!
34
+ kgio_wait_readable(timeout) or unreadable_socket!(timeout)
35
35
  else
36
36
  return rc
37
37
  end while true
@@ -42,7 +42,8 @@ class MogileFS::Socket < Kgio::Socket
42
42
  expect = buf.bytesize
43
43
  case rc = kgio_trywrite(buf)
44
44
  when :wait_writable
45
- kgio_wait_writable(timeout) or request_truncated!(written, expect)
45
+ kgio_wait_writable(timeout) or
46
+ request_truncated!(written, expect, timeout)
46
47
  when String
47
48
  written += expect - rc.bytesize
48
49
  buf = rc