mogilefs-client 3.0.0 → 3.1.0

data/lib/mogilefs/new_file/common.rb

@@ -0,0 +1,89 @@
+# -*- encoding: binary -*-
+# here are internal implementation details, do not use them in your code
+require 'socket'
+require 'uri'
+require 'digest/md5'
+require 'mogilefs/chunker'
+
+module MogileFS::NewFile::Common
+  # :stopdoc:
+  class RetryableError < MogileFS::Error; end
+  class EmptyResponseError < RetryableError; end
+  class BadResponseError < RetryableError; end
+  class UnparseableResponseError < RetryableError; end
+  class NoStorageNodesError < MogileFS::Error
+    def message; 'Unable to open socket to storage node'; end
+  end
+  class NonRetryableError < MogileFS::Error; end
+
+  MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
+
+  def read_response(sock)
+    tout = @opts[:new_file_max_time] || 3600.0
+    start_time = @opts[:start_time] and tout -= Time.now - start_time
+    case line = sock.timed_read(23, "", tout > 0.0 ? tout : 0)
+    when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
+    when nil
+      raise EmptyResponseError, 'Unable to read response line from server'
+    when %r{^HTTP/\d\.\d\s+(\d+)}
+      raise BadResponseError, "HTTP response status from upload: #$1"
+    else
+      raise UnparseableResponseError,
+            "Response line not understood: #{line.inspect}"
+    end
+  end
+
+  def create_close(devid, uri, bytes_uploaded)
+    args = {
+      :fid => @opts[:fid],
+      :devid => devid,
+      :key => @opts[:key],
+      :domain => @opts[:domain],
+      :size => bytes_uploaded,
+      :path => uri.to_s,
+    }
+    if @md5
+      args[:checksum] = "MD5:#{@md5.hexdigest}"
+    elsif String === @opts[:content_md5]
+      hex = @opts[:content_md5].unpack('m')[0].unpack('H*')[0]
+      args[:checksum] = "MD5:#{hex}"
+    end
+    args[:checksumverify] = 1 if @opts[:checksumverify]
+    backend = @opts[:backend]
+
+    # upload could've taken a long time, ping and try to ensure socket
+    # is valid to minimize (but not completely eliminate) the chance
+    # create_close hits a stale socket (while reading the response after
+    # writing to it) and becomes non-retryable.  We treat create_close
+    # specially as its less idempotent than any other command
+    # (even other non-idempotent ones).  There may be no hope of retrying
+    # the upload at all if data was streamed and calling create_close
+    # twice will hurt us...
+    backend.noop
+
+    backend.create_close(args)
+    bytes_uploaded
+  end
+
+  # aggressive keepalive settings on Linux + Ruby 1.9.2+
+  TCP_KEEPALIVE = {
+    :TCP_KEEPIDLE => 60, # seconds time before keepalive packet is sent
+    :TCP_KEEPINTVL => 5,
+    :TCP_KEEPCNT => 2,  # number of retries
+  }
+
+  req_consts = TCP_KEEPALIVE.keys
+  if (Socket.constants & req_consts).size == req_consts.size
+    def set_socket_options(sock)
+      sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1)
+      TCP_KEEPALIVE.each do |k,v|
+        sock.setsockopt(:IPPROTO_TCP, k, v)
+      end
+    end
+  else
+    def set_socket_options(sock)
+      sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1)
+    end
+  end
+  # :startdoc:
+end

data/lib/mogilefs/new_file/content_range.rb

@@ -0,0 +1,105 @@
+# -*- encoding: binary -*-
+# here are internal implementation details, do not rely on them in your code
+require 'net/http'
+require 'mogilefs/new_file/writer'
+
+# an IO-like object
+class MogileFS::NewFile::ContentRange
+  include MogileFS::NewFile::Writer
+  include MogileFS::NewFile::Common
+  attr_reader :md5
+
+  # :stopdoc:
+  begin
+    require 'net/http/persistent'
+    NHP = Net::HTTP::Persistent.new('mogilefs')
+
+    def hit(uri, req)
+      NHP.request(uri, req).value
+    end
+  rescue LoadError
+    def hit(uri, req)
+      Net::HTTP.start(uri.host, uri.port) { |h| h.request(req).value }
+    end
+  end
+  # :startdoc:
+
+  def initialize(dests, opts) # :nodoc:
+    @dests = dests
+    @opts = opts
+    @devid = @uri = @md5 = nil
+    @bytes_uploaded = 0
+    @errors = []
+  end
+
+  def get_dest # :nodoc:
+    return [ @devid, @uri ] if @uri
+    rv = @dests.shift or no_nodes!
+    rv[1] = URI.parse(rv[1])
+    rv
+  end
+
+  def no_nodes! # :nodoc:
+    raise NoStorageNodesError,
+          "all paths failed with PUT: #{@errors.join(', ')}", []
+  end
+
+  def request_for(uri, buf) # :nodoc:
+    put = Net::HTTP::Put.new(uri.path)
+    put["Content-Type"] = "application/octet-stream"
+    put["Content-MD5"] = [ Digest::MD5.digest(buf) ].pack("m").chomp!
+    if @bytes_uploaded > 0
+      last_byte = @bytes_uploaded + buf.bytesize - 1
+      put["Content-Range"] = "bytes #@bytes_uploaded-#{last_byte}/*"
+    end
+    put.body = buf
+
+    put
+  end
+
+  # see IO#write
+  def write(buf)
+    buf = String buf
+    len = buf.bytesize
+    return 0 if 0 == len
+
+    devid, uri = get_dest
+    put = request_for(uri, buf)
+    begin
+      hit(uri, put) # raises on error
+    rescue => e
+      raise if @bytes_uploaded > 0
+
+      # nothing uploaded, try another dest
+      @errors << "#{uri.to_s} - #{e.message} (#{e.class})"
+      devid, uri = get_dest
+      put = request_for(uri, buf)
+      retry
+    end
+
+    @uri, @devid = uri, devid if 0 == @bytes_uploaded
+    @bytes_uploaded += len
+    len
+  end
+
+  # called on close, do not use
+  def commit # :nodoc:
+    zero_byte_special if @bytes_uploaded == 0
+
+    create_close(@devid, @uri, @bytes_uploaded)
+  end
+
+  # special case for zero-byte files :<
+  def zero_byte_special # :nodoc:
+    @devid, @uri = get_dest
+    put = request_for(@uri, "")
+    begin
+      hit(@uri, put) # raises on error
+    rescue => e
+      @errors << "#{@uri.to_s} - #{e.message} (#{e.class})"
+      @devid, @uri = get_dest
+      put = request_for(@uri, "")
+      retry
+    end
+  end
+end

data/lib/mogilefs/new_file/stream.rb

@@ -0,0 +1,91 @@
+# -*- encoding: binary -*-
+# here are internal implementation details, do not use them in your code
+
+require 'mogilefs/new_file/writer'
+
+class MogileFS::NewFile::Stream
+  attr_reader :to_io
+  attr_reader :md5
+
+  include MogileFS::NewFile::Writer
+  include MogileFS::NewFile::Common
+
+  def initialize(dests, opts)
+    @opts = opts
+    @md5 = nil
+    @bytes_uploaded = 0
+    dests.each do |devid, path|
+      begin
+        uri = URI.parse(path)
+        sock = MogileFS::Socket.tcp(uri.host, uri.port)
+        set_socket_options(sock)
+        start_sock(sock, uri) # raise on errors
+        @to_io = sock
+        @uri = uri
+        @devid = devid
+        if ! @md5 && @opts[:content_length]
+          @writer = @to_io
+        else
+          @writer = MogileFS::Chunker.new(@to_io, @md5, @opts[:content_md5])
+        end
+        return
+      rescue SystemCallError => e
+        sock.close if sock && ! sock.closed?
+        errors ||= []
+        errors << "#{path} - #{e.message} (#{e.class})"
+      end
+    end
+
+    raise NoStorageNodesError,
+          "all paths failed with PUT: #{errors.join(', ')}", []
+  end
+
+  def write(buf)
+    buf = String buf
+    return 0 if 0 == buf.size
+    rv = @writer.write(buf)
+    @bytes_uploaded += rv
+    rv
+  end
+
+  def commit
+    @writer.flush
+
+    clen = @opts[:content_length]
+    if clen && @bytes_uploaded != clen
+      raise MogileFS::SizeMismatchError,
+           "did not upload expected content_length: #{clen} uploaded: " \
+           "#@bytes_uploaded"
+    end
+    read_response(@to_io) # raises on errors
+    create_close(@devid, @uri, @bytes_uploaded)
+    ensure
+      @to_io.close if @to_io && ! @to_io.closed?
+  end
+
+  def start_sock(sock, uri)
+    host_with_port = "#{uri.host}:#{uri.port}"
+    headers = "PUT #{uri.request_uri} HTTP/1.1\r\n" \
+              "Host: #{host_with_port}\r\n" \
+
+    content_md5 = @opts[:content_md5]
+    if String === content_md5
+      headers << "Content-MD5: #{content_md5}\r\n"
+    elsif content_md5.respond_to?(:call) ||
+          :trailer == content_md5 ||
+          MD5_TRAILER_NODES[host_with_port]
+      @md5 = Digest::MD5.new
+      headers << "Trailer: Content-MD5\r\n"
+    end
+
+    if ! @md5 && clen = @opts[:content_length]
+      headers << "Content-Length: #{clen}\r\n"
+    else
+      headers << "Transfer-Encoding: chunked\r\n"
+    end
+
+    sock.write(headers << "\r\n")
+  end
+
+  alias syswrite write
+end

data/lib/mogilefs/new_file/tempfile.rb

@@ -0,0 +1,24 @@
+# -*- encoding: binary -*-
+# here are internal implementation details, do not rely on them in your code
+require 'tempfile'
+require 'mogilefs/http_file'
+
+class MogileFS::NewFile::Tempfile < Tempfile
+  def initialize(*args)
+    @mogilefs_httpfile_args = args
+    super("mogilefs-client")
+    unlink
+  end
+
+  def commit
+    rewind
+    tmp = MogileFS::HTTPFile.new(*@mogilefs_httpfile_args)
+    tmp.big_io = to_io
+    tmp.commit
+  end
+
+  def close
+    commit
+    super
+  end
+end

data/lib/mogilefs/new_file/writer.rb

@@ -0,0 +1,57 @@
+# -*- encoding: binary -*-
+#
+# All objects yielded or returned by MogileFS::MogileFS#new_file should
+# conform to this interface (based on existing IO methods).  These objects
+# should be considered write-only.
+module MogileFS::NewFile::Writer
+
+  # see IO#puts
+  def puts(*args)
+    args.each do |obj|
+      write(obj)
+      write("\n")
+    end
+    nil
+  end
+
+  # see IO#putc
+  def putc(ch)
+    write(ch.respond_to?(:chr) ? ch.chr : ch[0])
+    ch
+  end
+
+  # see IO#print
+  def print(*args)
+    args = [ $_ ] unless args[0]
+    write(args.shift)
+    args.each do |obj|
+      write(obj)
+      write($,) if $,
+    end
+    write($\) if $\
+    nil
+  end
+
+  # see IO#printf
+  def printf(*args)
+    write(sprintf(*args))
+    nil
+  end
+
+  # see IO#<<
+  def <<(str)
+    write(str)
+    self
+  end
+
+  # This will issue the +create_close+ command to the MogileFS tracker
+  # and finalize the creation of a new file.  This returns +nil+ on
+  # success and will raise IOError if called twice.  For non-streaming
+  # implementations, this will initiate and finalize the upload.
+  #
+  # see IO#close
+  def close
+    commit
+    nil
+  end
+end

data/lib/mogilefs/pool.rb

@@ -3,23 +3,30 @@ require 'thread'
 
 class MogileFS::Pool
 
+  # Must be a positive Integer that is greater than :purge_keep
+  # Default: 5
+  attr_accessor :purge_threshold
+
+  # Must be a positive Integer that is smaller than :purge_threshold
+  # Default: 2
+  attr_accessor :purge_keep
+
   class BadObjectError < RuntimeError; end
 
   def initialize(klass, *args)
     @args = args
     @klass = klass
     @queue = Queue.new
-    @objects = []
+    @objects = {}
+    @purge_threshold = 5
+    @purge_keep = 2
   end
 
   def get
-    begin
-      object = @queue.pop true
-    rescue ThreadError
-      object = @klass.new(*@args)
-      @objects << object
-    end
-    object
+    @queue.pop true
+  rescue ThreadError
+    object = @klass.new(*@args)
+    @objects[object] = object
   end
 
   def put(o)
@@ -38,11 +45,12 @@ class MogileFS::Pool
   end
 
   def purge
-    return if @queue.length < 5
+    return if @queue.length < @purge_threshold
     begin
-      until @queue.length <= 2 do
+      until @queue.length <= @purge_keep
         obj = @queue.pop true
         @objects.delete obj
+        obj.backend.shutdown if MogileFS::Client === obj
       end
     rescue ThreadError
     end

data/lib/mogilefs/socket/kgio.rb

@@ -22,7 +22,7 @@ class MogileFS::Socket < Kgio::Socket
   def timed_read(len, dst = "", timeout = 5)
     case rc = kgio_tryread(len, dst)
     when :wait_readable
-      kgio_wait_readable(timeout) or unreadable_socket!
+      kgio_wait_readable(timeout) or unreadable_socket!(timeout)
     else
       return rc
     end while true
@@ -31,7 +31,7 @@ class MogileFS::Socket < Kgio::Socket
   def timed_peek(len, dst, timeout = 5)
     case rc = kgio_trypeek(len, dst)
     when :wait_readable
-      kgio_wait_readable(timeout) or unreadable_socket!
+      kgio_wait_readable(timeout) or unreadable_socket!(timeout)
     else
       return rc
     end while true
@@ -42,7 +42,8 @@ class MogileFS::Socket < Kgio::Socket
     expect = buf.bytesize
     case rc = kgio_trywrite(buf)
     when :wait_writable
-      kgio_wait_writable(timeout) or request_truncated!(written, expect)
+      kgio_wait_writable(timeout) or
+        request_truncated!(written, expect, timeout)
     when String
       written += expect - rc.bytesize
       buf = rc