mogilefs-client 1.3.1 → 2.0.0

data/lib/mogilefs/backend.rb

@@ -1,6 +1,6 @@
-require 'socket'
-require 'thread'
 require 'mogilefs'
+require 'mogilefs/util'
+require 'thread'
 
 ##
 # MogileFS::Backend communicates with the MogileFS trackers.
@@ -18,17 +18,29 @@ class MogileFS::Backend
     end
   end
 
+  BACKEND_ERRORS = {}
+
+  # this converts an error code from a mogilefsd tracker to an exception:
+  #
+  # Examples of some exceptions that get created:
+  #   class AfterMismatchError < MogileFS::Error; end
+  #   class DomainNotFoundError < MogileFS::Error; end
+  #   class InvalidCharsError < MogileFS::Error; end
+  def self.add_error(err_snake)
+    err_camel = err_snake.gsub(/(?:^|_)([a-z])/) { $1.upcase } << 'Error'
+    unless self.const_defined?(err_camel)
+      self.class_eval("class #{err_camel} < MogileFS::Error; end")
+    end
+    BACKEND_ERRORS[err_snake] = self.const_get(err_camel)
+  end
+
   ##
   # The last error
-  #--
-  # TODO Use Exceptions
 
   attr_reader :lasterr
 
   ##
   # The string attached to the last error
-  #--
-  # TODO Use Exceptions
 
   attr_reader :lasterrstr
 
@@ -61,8 +73,10 @@ class MogileFS::Backend
   # Closes this backend's socket.
 
   def shutdown
-    @socket.close unless @socket.nil? or @socket.closed?
-    @socket = nil
+    if @socket
+      @socket.close rescue nil # ignore errors
+      @socket = nil
+    end
   end
 
   # MogileFS::MogileFS commands
@@ -76,7 +90,7 @@ class MogileFS::Backend
   add_command :list_keys
 
   # MogileFS::Backend commands
-  
+
   add_command :get_hosts
   add_command :get_devices
   add_command :list_fids
@@ -92,14 +106,44 @@ class MogileFS::Backend
   add_command :delete_host
   add_command :set_state
 
-  private unless defined? $TESTING
+  # Errors copied from MogileFS/Worker/Query.pm
+  add_error 'dup'
+  add_error 'after_mismatch'
+  add_error 'bad_params'
+  add_error 'class_exists'
+  add_error 'class_has_files'
+  add_error 'class_not_found'
+  add_error 'db'
+  add_error 'domain_has_files'
+  add_error 'domain_exists'
+  add_error 'domain_not_empty'
+  add_error 'domain_not_found'
+  add_error 'failure'
+  add_error 'host_exists'
+  add_error 'host_mismatch'
+  add_error 'host_not_empty'
+  add_error 'host_not_found'
+  add_error 'invalid_chars'
+  add_error 'invalid_checker_level'
+  add_error 'invalid_mindevcount'
+  add_error 'key_exists'
+  add_error 'no_class'
+  add_error 'no_devices'
+  add_error 'no_domain'
+  add_error 'no_host'
+  add_error 'no_ip'
+  add_error 'no_key'
+  add_error 'no_port'
+  add_error 'none_match'
+  add_error 'plugin_aborted'
+  add_error 'state_too_high'
+  add_error 'unknown_command'
+  add_error 'unknown_host'
+  add_error 'unknown_key'
+  add_error 'unknown_state'
+  add_error 'unreg_domain'
 
-  ##
-  # Returns a new TCPSocket connected to +port+ on +host+.
-
-  def connect_to(host, port)
-    return TCPSocket.new(host, port)
-  end
+  private unless defined? $TESTING
 
   ##
   # Performs the +cmd+ request with +args+.
@@ -111,17 +155,18 @@ class MogileFS::Backend
       begin
         bytes_sent = socket.send request, 0
       rescue SystemCallError
-        @socket = nil
-        raise "couldn't connect to mogilefsd backend"
+        shutdown
+        raise MogileFS::UnreachableBackendError
       end
 
       unless bytes_sent == request.length then
-        raise "request truncated (sent #{bytes_sent} expected #{request.length})"
+        raise MogileFS::RequestTruncatedError,
+          "request truncated (sent #{bytes_sent} expected #{request.length})"
       end
 
       readable?
 
-      return parse_response(socket.gets)
+      parse_response(socket.gets)
     end
   end
 
@@ -129,7 +174,15 @@ class MogileFS::Backend
   # Makes a new request string for +cmd+ and +args+.
 
   def make_request(cmd, args)
-    return "#{cmd} #{url_encode args}\r\n"
+    "#{cmd} #{url_encode args}\r\n"
+  end
+
+  # this converts an error code from a mogilefsd tracker to an exception
+  # Most of these exceptions should already be defined, but since the
+  # MogileFS server code is liable to change and we may not always be
+  # able to keep up with the changes
+  def error(err_snake)
+    BACKEND_ERRORS[err_snake] || self.class.add_error(err_snake)
   end
 
   ##
@@ -140,25 +193,41 @@ class MogileFS::Backend
     if line =~ /^ERR\s+(\w+)\s*(.*)/ then
       @lasterr = $1
       @lasterrstr = $2 ? url_unescape($2) : nil
+      raise error(@lasterr)
       return nil
     end
 
     return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)/
 
-    raise "Invalid response from server: #{line.inspect}"
+    raise MogileFS::InvalidResponseError,
+          "Invalid response from server: #{line.inspect}"
   end
 
   ##
   # Raises if the socket does not become readable in +@timeout+ seconds.
 
   def readable?
-    found = select [socket], nil, nil, @timeout
-    if found.nil? or found.empty? then
-      peer = (@socket ? "#{@socket.peeraddr[3]}:#{@socket.peeraddr[1]} " : nil)
-      socket.close  # we DO NOT want the response we timed out waiting for, to crop up later on, on the same socket, intersperesed with a subsequent request! so, we close the socket if it times out like this
-      raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
+    timeleft = @timeout
+    peer = nil
+    loop do
+      t0 = Time.now
+      found = IO.select([socket], nil, nil, timeleft)
+      return true if found && found[0]
+      timeleft -= (Time.now - t0)
+
+      if timeleft < 0
+        peer = @socket ? "#{@socket.mogilefs_peername} " : nil
+
+        # we DO NOT want the response we timed out waiting for, to crop up later
+        # on, on the same socket, intersperesed with a subsequent request! so,
+        # we close the socket if it times out like this
+        shutdown
+        raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
+        break
+      end
+      shutdown
     end
-    return true
+    false
   end
 
   ##
@@ -173,8 +242,8 @@ class MogileFS::Backend
       next if @dead.include? host and @dead[host] > now - 5
 
       begin
-        @socket = connect_to(*host.split(':'))
-      rescue SystemCallError
+        @socket = Socket.mogilefs_new(*(host.split(/:/) << @timeout))
+      rescue SystemCallError, MogileFS::Timeout
         @dead[host] = now
         next
       end
@@ -182,25 +251,23 @@ class MogileFS::Backend
       return @socket
     end
 
-    raise "couldn't connect to mogilefsd backend"
+    raise MogileFS::UnreachableBackendError
   end
 
   ##
   # Turns a url params string into a Hash.
 
   def url_decode(str)
-    pairs = str.split('&').map do |pair|
-      pair.split('=', 2).map { |v| url_unescape v }
-    end
-
-    return Hash[*pairs.flatten]
+    Hash[*(str.split(/&/).map { |pair|
+      pair.split(/=/, 2).map { |x| url_unescape(x) }
+    } ).flatten]
   end
 
   ##
   # Turns a Hash (or Array of pairs) into a url params string.
 
   def url_encode(params)
-    return params.map do |k,v|
+    params.map do |k,v|
       "#{url_escape k.to_s}=#{url_escape v.to_s}"
     end.join("&")
   end
@@ -209,14 +276,14 @@ class MogileFS::Backend
   # Escapes naughty URL characters.
 
   def url_escape(str)
-    return str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
+    str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
   end
 
   ##
   # Unescapes naughty URL characters.
 
   def url_unescape(str)
-    return str.gsub(/%([a-f0-9][a-f0-9])/i) { [$1.to_i(16)].pack 'C' }.tr('+', ' ')
+    str.gsub(/%([a-f0-9][a-f0-9])/i) { [$1.to_i(16)].pack 'C' }.tr('+', ' ')
   end
 
 end

data/lib/mogilefs/bigfile.rb

@@ -0,0 +1,153 @@
+require 'zlib'
+require 'digest/md5'
+require 'uri'
+require 'mogilefs/util'
+
+module MogileFS::Bigfile
+  GZIP_HEADER = "\x1f\x8b".freeze # mogtool(1) has this
+  # VALID_TYPES = %w(file tarball partition).map { |x| x.freeze }.freeze
+
+  # returns a big_info hash if successful
+  def bigfile_stat(key)
+    parse_info(get_file_data(key))
+  end
+
+  # returns total bytes written and the big_info hash if successful, raises an
+  # exception if not wr_io is expected to be an IO-like object capable of
+  # receiving the syswrite method.
+  def bigfile_write(key, wr_io, opts = { :verify => false })
+    info = bigfile_stat(key)
+    zi = nil
+    md5 = opts[:verify] ? Digest::MD5.new : nil
+    total = 0
+
+    # we only decode raw zlib deflated streams that mogtool (unfortunately)
+    # generates.  tarballs and gzip(1) are up to to the application to decrypt.
+    filter = Proc.new do |buf|
+      if zi == nil
+        if info[:compressed] && info[:type] == 'file' &&
+             buf.length >= 2 && buf[0,2] != GZIP_HEADER
+          zi = Zlib::Inflate.new
+
+          # mogtool(1) seems to have a bug that causes it to generate bogus
+          # MD5s if zlib deflate is used.  Don't trust those MD5s for now...
+          md5 = nil
+        else
+          zi = false
+        end
+      end
+      buf ||= ''
+      if zi
+        zi.inflate(buf)
+      else
+        md5 << buf
+        buf
+      end
+    end if (info[:compressed] || md5)
+
+    info[:parts].each_with_index do |part,part_nr|
+      next if part_nr == 0 # info[:parts][0] is always empty
+      uris = verify_uris(part[:paths].map { |path| URI.parse(path) })
+      if uris.empty?
+        # part[:paths] may not be valid anymore due to rebalancing, however we
+        # can get_keys on key,<part_nr> and retry paths if all paths fail
+        part[:paths] = get_paths("#{key.gsub(/^big_info:/, '')},#{part_nr}")
+        uris = verify_uris(part[:paths].map { |path| URI.parse(path) })
+        raise MogileFS::Backend::NoDevices if uris.empty?
+      end
+
+      sock = http_get_sock(uris[0])
+      md5.reset if md5
+      w = sysrwloop(sock, wr_io, filter)
+
+      if md5 && md5.hexdigest != part[:md5]
+        raise MogileFS::ChecksumMismatchError, "#{md5} != #{part[:md5]}"
+      end
+      total += w
+    end
+
+    syswrite_full(wr_io, zi.finish) if zi
+
+    [ total, info ]
+  end
+
+  private
+
+    include MogileFS::Util
+
+    ##
+    # parses the contents of a _big_info: string or IO object
+    def parse_info(info = '')
+      rv = { :parts => [] }
+      info.each_line do |line|
+        line.chomp!
+        case line
+        when /^(des|type|filename)\s+(.+)$/
+          rv[$1.to_sym] = $2
+        when /^compressed\s+([01])$/
+          rv[:compressed] = ($1 == '1')
+        when /^(chunks|size)\s+(\d+)$/
+          rv[$1.to_sym] = $2.to_i
+        when /^part\s+(\d+)\s+bytes=(\d+)\s+md5=(.+)\s+paths:\s+(.+)$/
+          rv[:parts][$1.to_i] = {
+            :bytes => $2.to_i,
+            :md5 => $3.downcase,
+            :paths => $4.split(/\s*,\s*/),
+          }
+        end
+      end
+
+      rv
+    end
+
+end # module MogileFS::Bigfile
+
+__END__
+# Copied from mogtool:
+# http://code.sixapart.com/svn/mogilefs/utils/mogtool, r1221
+
+# this is a temporary file that we delete when we're doing recording all chunks
+
+_big_pre:<key>
+
+    starttime=UNIXTIMESTAMP
+
+# when done, we write the _info file and delete the _pre.
+
+_big_info:<key>
+
+    des Cow's ljdb backup as of 2004-11-17
+    type  { partition, file, tarball }
+    compressed {0, 1}
+    filename  ljbinlog.305.gz
+    partblocks  234324324324
+
+
+    part 1 <bytes> <md5hex>
+    part 2 <bytes> <md5hex>
+    part 3 <bytes> <md5hex>
+    part 4 <bytes> <md5hex>
+    part 5 <bytes> <md5hex>
+
+_big:<key>,<n>
+_big:<key>,<n>
+_big:<key>,<n>
+
+
+Receipt format:
+
+BEGIN MOGTOOL RECIEPT
+type partition
+des Foo
+compressed foo
+
+part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
+part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
+part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
+part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
+
+
+END RECIEPT
+
+
+

data/lib/mogilefs/client.rb

@@ -38,8 +38,6 @@ class MogileFS::Client
 
   ##
   # The last error reported by the backend.
-  #--
-  # TODO use Exceptions
 
   def err
     @backend.lasterr
@@ -47,8 +45,6 @@ class MogileFS::Client
 
   ##
   # The last error message reported by the backend.
-  #--
-  # TODO use Exceptions
 
   def errstr
     @backend.lasterrstr
@@ -58,7 +54,7 @@ class MogileFS::Client
   # Is this a read-only client?
 
   def readonly?
-    return @readonly
+    @readonly
   end
 
 end

data/lib/mogilefs/httpfile.rb

@@ -1,5 +1,3 @@
-require 'fcntl'
-require 'socket'
 require 'stringio'
 require 'uri'
 require 'mogilefs/backend'
@@ -12,18 +10,23 @@ require 'mogilefs/util'
 # You really don't want to create an HTTPFile by hand.  Instead you want to
 # create a new file using MogileFS::MogileFS.new_file.
 #
-# WARNING! HTTP mode is completely untested as I cannot make it work on
-# FreeBSD.  Please send patches/tests if you find bugs.
 #--
 # TODO dup'd content in MogileFS::NFSFile
 
 class MogileFS::HTTPFile < StringIO
   include MogileFS::Util
 
+  class EmptyResponseError < MogileFS::Error; end
+  class BadResponseError < MogileFS::Error; end
+  class UnparseableResponseError < MogileFS::Error; end
+  class NoStorageNodesError < MogileFS::Error
+    def message; 'Unable to open socket to storage node'; end
+  end
+
   ##
-  # The path this file will be stored to.
+  # The URI this file will be stored to.
 
-  attr_reader :path
+  attr_reader :uri
 
   ##
   # The key for this file.  This key won't represent a real file until you've
@@ -37,9 +40,9 @@ class MogileFS::HTTPFile < StringIO
   attr_reader :class
 
   ##
-  # The bigfile name in case we have file > 256M
+  # The big_io name in case we have file > 256M
 
-  attr_accessor :bigfile
+  attr_accessor :big_io
 
   ##
   # Works like File.open.  Use MogileFS::MogileFS#new_file instead of this
@@ -61,93 +64,84 @@ class MogileFS::HTTPFile < StringIO
   # Creates a new HTTPFile with MogileFS-specific data.  Use
   # MogileFS::MogileFS#new_file instead of this method.
 
-  def initialize(mg, fid, path, devid, klass, key, dests, content_length)
+  def initialize(mg, fid, klass, key, dests, content_length)
     super ''
     @mg = mg
     @fid = fid
-    @path = path
-    @devid = devid
+    @uri = @devid = nil
     @klass = klass
     @key = key
-    @bigfile = nil
+    @big_io = nil
 
-    @dests = dests.map { |(_,u)| URI.parse u }
+    @dests = dests
     @tried = {}
 
     @socket = nil
   end
 
   ##
-  # Closes the file handle and marks it as closed in MogileFS.
-
-  def close
-    connect_socket
-
-    file_size = nil
-    if @bigfile
+  # Writes an HTTP PUT request to +sock+ to upload the file and
+  # returns file size if the socket finished writing
+  def upload(devid, uri)
+    file_size = length
+    sock = Socket.mogilefs_new(uri.host, uri.port)
+    sock.mogilefs_tcp_cork = true
+
+    if @big_io
       # Don't try to run out of memory
-      fp = File.open(@bigfile)
-      file_size = File.size(@bigfile)
-      @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-Length: #{file_size}\r\n\r\n"
-      sysrwloop(fp, @socket)
-      fp.close
+      File.open(@big_io) do |fp|
+        file_size = fp.stat.size
+        fp.sync = true
+        syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
+                            "Content-Length: #{file_size}\r\n\r\n")
+        sysrwloop(fp, sock)
+      end
     else
-      @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n#{string}"
+      syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
+                          "Content-Length: #{length}\r\n\r\n#{string}")
     end
+    sock.mogilefs_tcp_cork = false
+
+    line = sock.gets or
+      raise EmptyResponseError, 'Unable to read response line from server'
 
-    if connected? then
-      line = @socket.gets
-      raise 'Unable to read response line from server' if line.nil?
-
-      if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
-        status = Integer $1
-        case status
-        when 200..299 then # success!
-        else
-          raise "HTTP response status from upload: #{status}"
-        end
+    if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
+      case $1.to_i
+      when 200..299 then # success!
       else
-        raise "Response line not understood: #{line}"
+        raise BadResponseError, "HTTP response status from upload: #{$1}"
       end
-
-      @socket.close
+    else
+      raise UnparseableResponseError, "Response line not understood: #{line}"
     end
 
-    @mg.backend.create_close(:fid => @fid, :devid => @devid,
+    @mg.backend.create_close(:fid => @fid, :devid => devid,
                              :domain => @mg.domain, :key => @key,
-                             :path => @path, :size => length)
-    return file_size if @bigfile
-    return nil
-  end
+                             :path => uri.to_s, :size => file_size)
+    file_size
+  end # def upload
 
-  private
-
-  def connected?
-    return !(@socket.nil? or @socket.closed?)
-  end
-
-  def connect_socket
-    return @socket if connected?
-
-    next_path
-
-    if @path.nil? then
-      @tried.clear
-      next_path
-      raise 'Unable to open socket to storage node' if @path.nil?
-    end
-
-    @socket = TCPSocket.new @path.host, @path.port
-  end
-
-  def next_path
-    @path = nil
-    @dests.each do |dest|
-      unless @tried.include? dest then
-        @path = dest
-        return
+  def close
+    try_dests = @dests.dup
+    last_err = nil
+
+    loop do
+      devid, url = try_dests.shift
+      devid && url or break
+
+      uri = URI.parse(url)
+      begin
+        bytes = upload(devid, uri)
+        @devid, @uri = devid, uri
+        return bytes
+      rescue SystemCallError, Errno::ECONNREFUSED, MogileFS::Timeout,
+             EmptyResponseError, BadResponseError,
+             UnparseableResponseError => err
+        last_err = @tried[url] = err
       end
     end
+
+    raise last_err ? last_err : NoStorageNodesError
   end
 
 end