mogilefs-client 3.0.0 → 3.1.0

data/lib/mogilefs/client.rb

@@ -11,7 +11,9 @@ class MogileFS::Client
 
   attr_reader :backend
 
+  # :stopdoc:
   attr_accessor :hosts if defined? $TESTING
+  # :startdoc
 
   ##
   # Creates a new Client.  See MogileFS::Backend#initialize for how to specify
@@ -24,6 +26,7 @@ class MogileFS::Client
     @hosts = args[:hosts]
     @readonly = args[:readonly] ? true : false
     @timeout = args[:timeout]
+    @fail_timeout = args[:fail_timeout]
 
     reload
   end
@@ -32,7 +35,9 @@ class MogileFS::Client
   # Creates a new MogileFS::Backend.
 
   def reload
-    @backend = MogileFS::Backend.new :hosts => @hosts, :timeout => @timeout
+    @backend = MogileFS::Backend.new(:hosts => @hosts,
+                                     :timeout => @timeout,
+                                     :fail_timeout => @fail_timeout)
   end
 
   ##

data/lib/mogilefs/http_file.rb

@@ -1,8 +1,7 @@
 # -*- encoding: binary -*-
 # here are internal implementation details, do not use them in your code
 require 'stringio'
-require 'uri'
-require 'mogilefs/chunker'
+require 'mogilefs/new_file'
 
 ##
 # HTTPFile wraps up the new file operations for storing files onto an HTTP
@@ -12,49 +11,7 @@ require 'mogilefs/chunker'
 # create a new file using MogileFS::MogileFS.new_file.
 #
 class MogileFS::HTTPFile < StringIO
-  class EmptyResponseError < MogileFS::Error; end
-  class BadResponseError < MogileFS::Error; end
-  class UnparseableResponseError < MogileFS::Error; end
-  class NoStorageNodesError < MogileFS::Error
-    def message; 'Unable to open socket to storage node'; end
-  end
-  class NonRetryableError < MogileFS::Error; end
-
-  class HTTPSock < MogileFS::Socket
-    attr_accessor :start
-
-    # Increase timeout as we become more invested in uploading with
-    # this socket.  The server could be experiencing I/O delays
-    # from large uploads because the sysadmin forgot to tune the
-    # VM sysctls for handling large files.
-    def write(buf)
-      timed_write(buf, Time.now - @start + 5.0)
-    end
-  end
-
-  # :stopdoc:
-  MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
-  class << self
-    attr_accessor :response_timeout_cb
-  end
-
-  # temporary directories (nginx) may not be configured on the
-  # same device, necessitating a time-consuming full file copy
-  # instead of a quick rename(2)/link(2) operation
-  @response_timeout_cb = lambda do |elapsed_time, bytes_uploaded|
-    mbytes_uploaded = bytes_uploaded / (1024.0 * 1024.0)
-    # assumes worst case is 10M/s on the remote storage disk
-    t = mbytes_uploaded * 10 + elapsed_time
-    t < 5 ? 5 : t
-  end
-  # :startdoc:
-
-  ##
-  # The URI this file will be stored to.
-
-  attr_reader :uri
-
-  attr_reader :devid
+  include MogileFS::NewFile::Common
 
   ##
   # The big_io name in case we have file > 256M
@@ -67,34 +24,50 @@ class MogileFS::HTTPFile < StringIO
   # Creates a new HTTPFile with MogileFS-specific data.  Use
   # MogileFS::MogileFS#new_file instead of this method.
 
-  def initialize(dests, content_length)
+  def initialize(dests, opts = nil)
     super ""
-    @streaming_io = @big_io = @uri = @devid = @active = nil
+    @md5 = @streaming_io = @big_io = @active = nil
     @dests = dests
+    @opts = Integer === opts ? { :content_length => opts } : opts
   end
 
   def request_put(sock, uri, file_size, input = nil)
     host_with_port = "#{uri.host}:#{uri.port}"
-    md5 = false
-    if MD5_TRAILER_NODES[host_with_port]
+    clen = @opts[:content_length]
+    file_size ||= clen
+
+    content_md5 = @opts[:content_md5]
+    if String === content_md5
+      file_size or
+        raise ArgumentError,
+              ":content_length must be specified with :content_md5 String"
+      file_size = "#{file_size}\r\nContent-MD5: #{content_md5}"
+    elsif content_md5.respond_to?(:call) ||
+          :trailer == content_md5 ||
+          MD5_TRAILER_NODES[host_with_port]
       file_size = nil
-      md5 = true
+      @md5 = Digest::MD5.new
     end
 
     if file_size
       sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \
                  "Content-Length: #{file_size}\r\n\r\n")
-      input ? MogileFS.io.copy_stream(@active = input, sock) : yield(sock)
+      rv = input ? MogileFS.io.copy_stream(@active = input, sock) : yield(sock)
     else
-      trailers = md5 ? "Trailer: Content-MD5\r\n" : ""
+      trailers = @md5 ? "Trailer: Content-MD5\r\n" : ""
       sock.write("PUT #{uri.request_uri} HTTP/1.1\r\n" \
                  "Host: #{host_with_port}\r\n#{trailers}" \
                  "Transfer-Encoding: chunked\r\n\r\n")
-      tmp = MogileFS::Chunker.new(sock, md5)
+      tmp = MogileFS::Chunker.new(sock, @md5, content_md5)
       rv = input ? MogileFS.io.copy_stream(@active = input, tmp) : yield(tmp)
       tmp.flush
-      rv
     end
+
+    if clen && clen != rv
+      raise MogileFS::SizeMismatchError,
+            ":content_length expected: #{clen.inspect}, actual: #{rv.inspect}"
+    end
+    rv
   end
 
   def put_streaming_io(sock, uri) # unlikely to be used
@@ -121,51 +94,39 @@ class MogileFS::HTTPFile < StringIO
   # Writes an HTTP PUT request to +sock+ to upload the file and
   # returns file size if the socket finished writing
   def upload(devid, uri) # :nodoc:
-    start = Time.now
-    sock = HTTPSock.tcp(uri.host, uri.port)
-    sock.start = start
+    sock = MogileFS::Socket.tcp(uri.host, uri.port)
+    set_socket_options(sock)
     file_size = length
 
     if @streaming_io
       file_size = put_streaming_io(sock, uri)
     elsif @big_io
-      if String === @big_io || @big_io.respond_to?(:to_path)
+      stat = file = size = nil
+      if @big_io.respond_to?(:stat)
+        stat = @big_io.stat
+      elsif String === @big_io || @big_io.respond_to?(:to_path)
         file = File.open(@big_io)
         stat = file.stat
-        file_size = request_put(sock, uri, stat.file? ? stat.size : nil, file)
-      else
-        size = nil
-        if @big_io.respond_to?(:stat)
-          stat = @big_io.stat
-          size = stat.size if stat.file?
-        elsif @big_io.respond_to?(:size)
-          size = @big_io.size
-        end
-        file_size = request_put(sock, uri, size, @big_io)
+      elsif @big_io.respond_to?(:size)
+        size = @big_io.size
+      end
+      if stat && stat.file?
+        size ||= stat.size
+        file ||= @big_io.to_io if @big_io.respond_to?(:to_io)
       end
+      file_size = request_put(sock, uri, size, file || @big_io)
     else
       rewind
       request_put(sock, uri, file_size, self)
     end
 
-    tout = self.class.response_timeout_cb.call(Time.now - start, file_size)
-
-    case line = sock.timed_read(23, "", tout)
-    when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
-      file_size
-    when nil
-      raise EmptyResponseError, 'Unable to read response line from server'
-    when %r{^HTTP/\d\.\d\s+(\d+)}
-      raise BadResponseError, "HTTP response status from upload: #$1"
-    else
-      raise UnparseableResponseError,
-            "Response line not understood: #{line.inspect}"
-    end
-    rescue => err
+    read_response(sock) # raises on errors
+    file_size
+    rescue SystemCallError, RetryableError => err
       rewind_or_raise!(uri, err)
       raise
     ensure
-      file.close if file
+      file.close if file && @big_io != file
       sock.close if sock
   end
 
@@ -175,11 +136,8 @@ class MogileFS::HTTPFile < StringIO
       begin
         uri = URI.parse(path)
         bytes_uploaded = upload(devid, uri)
-        @devid, @uri = devid, uri
-        return bytes_uploaded
-      rescue NonRetryableError
-        raise
-      rescue => e
+        return create_close(devid, uri, bytes_uploaded)
+      rescue SystemCallError, RetryableError => e
         errors ||= []
         errors << "#{path} - #{e.message} (#{e.class})"
       end
@@ -188,4 +146,9 @@ class MogileFS::HTTPFile < StringIO
     raise NoStorageNodesError,
           "all paths failed with PUT: #{errors.join(', ')}", []
   end
+
+  def close
+    commit
+    super
+  end
 end

data/lib/mogilefs/mogilefs.rb

@@ -34,15 +34,41 @@ class MogileFS::MogileFS < MogileFS::Client
   # The domain of keys for this MogileFS client.
   attr_accessor :domain
 
-  # The timeout for get_file_data.  Defaults to five seconds.
+  # The timeout for get_file_data (per-read() system call).
+  # Defaults to five seconds.
   attr_accessor :get_file_data_timeout
 
+  # The maximum allowed time for creating a new_file.  Defaults to 1 hour.
+  attr_accessor :new_file_max_time
+
   # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
   # :domain specifying the domain of this client.
+  #
+  # Optional parameters for +args+:
+  #
+  # [:get_file_data_timeout => Numeric]
+  #
+  #   See get_file_data_timeout
+  #
+  # [:new_file_max_time => Numeric]
+  #
+  #   See new_file_max_time
+  #
+  # [:fail_timeout => Numeric]
+  #
+  #   Delay before retrying a failed tracker backends.
+  #   Defaults to 5 seconds.
+  #
+  # [:timeout => Numeric]
+  #
+  #   Timeout for tracker backend responses.
+  #   Defaults to 3 seconds.
+  #
   def initialize(args = {})
     @domain = args[:domain]
 
-    @get_file_data_timeout = 5
+    @get_file_data_timeout = args[:get_file_data_timeout] || 5
+    @new_file_max_time = args[:new_file_max_time] || 3600.0
 
     raise ArgumentError, "you must specify a domain" unless @domain
 
@@ -113,11 +139,8 @@ class MogileFS::MogileFS < MogileFS::Client
 
   # Returns +true+ if +key+ exists, +false+ if not
   def exist?(key)
-    rv = nil
-    args = { :key => key, :domain => @domain }
-    @backend.pipeline_dispatch(:get_paths, args) { |x| rv = (Hash === x) }
-    @backend.pipeline_wait(1)
-    rv
+    args = { :key => key, :domain => @domain , :ruby_no_raise => true}
+    Hash === @backend.get_paths(args)
   end
 
   # Get the URIs for +key+ (paths) as URI::HTTP objects
@@ -125,20 +148,45 @@ class MogileFS::MogileFS < MogileFS::Client
     get_paths(key, *args).map! { |path| URI.parse(path) }
   end
 
-  # Creates a new file +key+ in +klass+.  +bytes+ is currently unused.
-  # Consider using store_file instead of this method for large files.
-  # This requires a block passed to it and operates like File.open.
-  # This atomically replaces existing data stored as +key+ when
-  def new_file(key, klass = nil, bytes = 0) # :yields: file
+  # Creates a new file +key+ in the domain of this object.
+  #
+  # +bytes+ is the expected size of the file if known in advance
+  #
+  # It operates like File.open(..., "w") and may take an optional
+  # block, yielding an IO-like object with support for the methods
+  # documented in MogileFS::NewFile::Writer.
+  #
+  # This atomically replaces existing data stored as +key+
+  # when the block exits or when the returned object is closed.
+  #
+  # +args+ may contain the following options:
+  #
+  # [:content_length => Integer]
+  #
+  #   This has the same effect as the (deprecated) +bytes+ parameter.
+  #
+  # [ :largefile => :stream, :content_range or :tempfile ]
+  #
+  #   See MogileFS::NewFile for more information on this
+  #
+  # [ :class => String]
+  #
+  #   The MogileFS storage class of the object.
+  def new_file(key, args = nil, bytes = nil) # :yields: file
     raise MogileFS::ReadOnlyError if readonly?
-    opts = { :domain => @domain, :key => key, :multi_dest => 1 }
-    opts[:class] = klass if klass && klass != "default"
+    opts = { :key => key, :multi_dest => 1 }
+    case args
+    when Hash
+      opts[:domain] = args[:domain]
+      klass = args[:class] and "default" != klass and opts[:class] = klass
+    when String
+      opts[:class] = args if "default" != args
+    end
+    opts[:domain] ||= @domain
     res = @backend.create_open(opts)
 
     dests = if dev_count = res['dev_count'] # multi_dest succeeded
-      (1..dev_count.to_i).map do |i|
-        [res["devid_#{i}"], res["path_#{i}"]]
-      end
+      (1..dev_count.to_i).map { |i| [res["devid_#{i}"], res["path_#{i}"]] }
     else # single destination returned
       # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
       # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
@@ -149,19 +197,23 @@ class MogileFS::MogileFS < MogileFS::Client
       [[res['devid'], res['path']]]
     end
 
+    opts.merge!(args) if Hash === args
+    opts[:backend] = @backend
+    opts[:fid] = res['fid']
+    opts[:content_length] ||= bytes if bytes
+    opts[:new_file_max_time] ||= @new_file_max_time
+    opts[:start_time] = Time.now
+
     case (dests[0][1] rescue nil)
-    when /^http:\/\// then
-      http_file = MogileFS::HTTPFile.new(dests, bytes)
-      yield http_file
-      rv = http_file.commit
-      @backend.create_close(:fid => res['fid'],
-                            :devid => http_file.devid,
-                            :domain => @domain,
-                            :key => key,
-                            :path => http_file.uri.to_s,
-                            :size => rv)
-      rv
-    when nil, '' then
+    when %r{\Ahttp://}
+      http_file = MogileFS::NewFile.new(dests, opts)
+      if block_given?
+        yield http_file
+        return http_file.commit # calls create_close
+      else
+        return http_file
+      end
+    when nil, ''
       raise MogileFS::EmptyPathError,
             "Empty path for mogile upload res=#{res.inspect}"
     else
@@ -174,18 +226,20 @@ class MogileFS::MogileFS < MogileFS::Client
   # either a path name (String or Pathname object) or an IO-like object that
   # responds to #read or #readpartial.  Returns size of +file+ stored.
   # This atomically replaces existing data stored as +key+
-  def store_file(key, klass, file)
+  def store_file(key, klass, file, opts = nil)
     raise MogileFS::ReadOnlyError if readonly?
+    (opts ||= {})[:class] = klass if String === klass
 
-    new_file(key, klass) { |mfp| mfp.big_io = file }
+    new_file(key, opts) { |mfp| mfp.big_io = file }
   end
 
   # Stores +content+ into +key+ in class +klass+, where +content+ is a String
   # This atomically replaces existing data stored as +key+
-  def store_content(key, klass, content)
+  def store_content(key, klass, content, opts = nil)
     raise MogileFS::ReadOnlyError if readonly?
+    (opts ||= {})[:class] = klass if String === klass
 
-    new_file key, klass do |mfp|
+    new_file(key, opts) do |mfp|
       if content.is_a?(MogileFS::Util::StoreContent)
         mfp.streaming_io = content
       else
@@ -236,12 +290,10 @@ class MogileFS::MogileFS < MogileFS::Client
     @backend.respond_to?(:_list_keys) and
       return @backend._list_keys(domain, prefix, after, limit, &block)
 
-    begin
-      res = @backend.list_keys(:domain => domain, :prefix => prefix,
-                               :after => after, :limit => limit)
-    rescue MogileFS::Backend::NoneMatchError
-      return
-    end
+    res = @backend.list_keys(:domain => domain, :prefix => prefix,
+                             :after => after, :limit => limit,
+                             :ruby_no_raise => true)
+    MogileFS::Backend::NoneMatchError === res and return
 
     keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
     if block

data/lib/mogilefs/new_file.rb

@@ -0,0 +1,78 @@
+# -*- encoding: binary -*-
+#
+# The MogileFS::MogileFS#new_file method is enhanced in v3.1.0+
+# to support the :largefile parameter.  While we have always
+# supported large files via the "store_file" method, streaming
+# large amounts of content of an unknown length required the use
+# of awkward APIs.
+#
+# It is possible to stream large content of known length any WebDAV server.
+# One example of this is for mirroring a large file from an existing HTTP
+# server to \MogileFS without letting it hit the local filesystem.
+#
+#   uri = URI('http://example.com/large_file')
+#   Net::HTTP.start(uri.host, uri.port) do |http|
+#     req = Net::HTTP::Get.new(uri.request_uri)
+#
+#     http.request(req) do |response|
+#       if len = response.content_length
+#         io = mg.new_file('key', :largefile => true, :content_length => len)
+#       else
+#         warn "trying to upload with Transfer-Encoding: chunked"
+#         warn "this is not supported by all WebDAV servers"
+#         io = mg.new_file('key', :largefile => :stream)
+#       end
+#       response.read_body { |buf| io.write(buf) }
+#       io.close
+#     end
+#   end
+#
+# If your WebDAV servers have chunked PUT support (e.g. Perlbal), you can
+# stream a file of unknown length using "Transfer-Encoding: chunked".
+#
+#   nf = mg.new_file("key", :largefile => :stream)
+#   nf.write "hello"
+#   nf.write ...
+#   nf.close
+#
+# If your WebDAV server has partial PUT support (e.g Apache), you can
+# you can use multiple PUT requests with "Content-Range" support.
+# This method is slower than Transfer-Encoding: chunked.
+#
+#   nf = mg.new_file("key", :largefile => :content_range)
+#   nf.write "hello"
+#   nf.write ...
+#   nf.close
+#
+# Finally, if your WebDAV servers does not support either partial nor
+# nor chunked PUTs, you must buffer a large file of unknown length
+# using a Tempfile:
+#
+#   nf = mg.new_file("key", :largefile => :tempfile)
+#   nf.write "hello"
+#   nf.write ...
+#   nf.close
+#
+module MogileFS::NewFile
+
+  # avoiding autoload for new code since it's going away in Ruby...
+  def self.new(dests, opts) # :nodoc:
+    largefile = opts[:largefile]
+    largefile = :stream if largefile && opts[:content_length]
+    require "mogilefs/new_file/#{largefile}" if Symbol === largefile
+    case largefile
+    when nil, false
+      MogileFS::HTTPFile
+    when :stream
+      Stream
+    when :content_range
+      ContentRange
+    when :tempfile
+      Tempfile
+    else
+      raise ArgumentError, "largefile: #{largefile.inspect} not understood"
+    end.new(dests, opts)
+  end
+end
+
+require 'mogilefs/new_file/common'