mogilefs-client 2.2.0 → 3.0.0.rc1

data/lib/mogilefs/backend.rb

@@ -1,25 +1,31 @@
 # -*- encoding: binary -*-
-require 'mogilefs'
-require 'mogilefs/util'
 require 'thread'
 
-##
-# MogileFS::Backend communicates with the MogileFS trackers.
-
+# This class communicates with the MogileFS trackers.
+# You should not have to use this directly unless you are developing
+# support for new commands or plugins for MogileFS
 class MogileFS::Backend
 
-  ##
   # Adds MogileFS commands +names+.
-
   def self.add_command(*names)
     names.each do |name|
       define_method name do |*args|
-        do_request name, args.first || {}
+        do_request(name, args[0] || {}, false)
+      end
+    end
+  end
+
+  # adds idempotent MogileFS commands +names+, these commands may be retried
+  # transparently on a different tracker if there is a network/server error.
+  def self.add_idempotent_command(*names)
+    names.each do |name|
+      define_method name do |*args|
+        do_request(name, args[0] || {}, true)
       end
     end
   end
 
-  BACKEND_ERRORS = {}
+  BACKEND_ERRORS = {} # :nodoc:
 
   # this converts an error code from a mogilefsd tracker to an exception:
   #
@@ -30,10 +36,10 @@ class MogileFS::Backend
   def self.add_error(err_snake)
     err_camel = err_snake.gsub(/(?:^|_)([a-z])/) { $1.upcase }
     err_camel << 'Error' unless /Error\z/ =~ err_camel
-    unless self.const_defined?(err_camel)
-      self.class_eval("class #{err_camel} < MogileFS::Error; end")
+    unless const_defined?(err_camel)
+      const_set(err_camel, Class.new(MogileFS::Error))
     end
-    BACKEND_ERRORS[err_snake] = self.const_get(err_camel)
+    BACKEND_ERRORS[err_snake] = const_get(err_camel)
   end
 
   ##
@@ -67,6 +73,7 @@ class MogileFS::Backend
     @socket = nil
     @lasterr = nil
     @lasterrstr = nil
+    @pending = []
 
     @dead = {}
   end
@@ -82,19 +89,21 @@ class MogileFS::Backend
 
   add_command :create_open
   add_command :create_close
-  add_command :get_paths
+  add_idempotent_command :get_paths
   add_command :delete
-  add_command :sleep
+  add_idempotent_command :sleep
   add_command :rename
-  add_command :list_keys
+  add_idempotent_command :list_keys
+  add_idempotent_command :file_info
+  add_idempotent_command :file_debug
 
   # MogileFS::Backend commands
 
-  add_command :get_hosts
-  add_command :get_devices
-  add_command :list_fids
-  add_command :stats
-  add_command :get_domains
+  add_idempotent_command :get_hosts
+  add_idempotent_command :get_devices
+  add_idempotent_command :list_fids
+  add_idempotent_command :stats
+  add_idempotent_command :get_domains
   add_command :create_domain
   add_command :delete_domain
   add_command :create_class
@@ -104,6 +113,7 @@ class MogileFS::Backend
   add_command :update_host
   add_command :delete_host
   add_command :set_state
+  add_command :replicate_now
 
   # Errors copied from MogileFS/Worker/Query.pm
   add_error 'dup'
@@ -143,51 +153,133 @@ class MogileFS::Backend
   add_error 'unknown_state'
   add_error 'unreg_domain'
 
-  private unless defined? $TESTING
-
-  # record-separator for mogilefsd responses, update this if the protocol
-  # changes
-  RS = "\n"
-
-  def shutdown_unlocked # :nodoc:
+  def shutdown_unlocked(do_raise = false) # :nodoc:
+    @pending = []
     if @socket
       @socket.close rescue nil # ignore errors
       @socket = nil
     end
+    raise if do_raise
   end
 
-  ##
-  # Performs the +cmd+ request with +args+.
+  def dispatch_unlocked(request, timeout = @timeout) # :nodoc:
+    begin
+      io = socket
+      io.timed_write(request, timeout)
+      io
+    rescue SystemCallError, MogileFS::RequestTruncatedError  => err
+      @dead[@active_host] = [ Time.now, err ]
+      shutdown_unlocked
+      retry
+    end
+  end
+
+  def pipeline_gets_unlocked(io, timeout) # :nodoc:
+    line = io.timed_gets(timeout) or
+      raise MogileFS::PipelineError,
+            "EOF with #{@pending.size} requests in-flight"
+    ready = @pending.shift
+    ready[1].call(parse_response(line, ready[0]))
+  end
+
+  def timeout_update(timeout, t0) # :nodoc:
+    timeout -= (Time.now - t0)
+    timeout < 0 ? 0 : timeout
+  end
+
+  # try to read any responses we have pending already before filling
+  # the pipeline more requests.  This usually takes very little time,
+  # but trackers may return huge responses and we could be on a slow
+  # network.
+  def pipeline_drain_unlocked(io, timeout) # :nodoc:
+    set = [ io ]
+    while @pending.size > 0
+      t0 = Time.now
+      r = IO.select(set, set, nil, timeout)
+      timeout = timeout_update(timeout, t0)
+
+      if r && r[0][0]
+        t0 = Time.now
+        pipeline_gets_unlocked(io, timeout)
+        timeout = timeout_update(timeout, t0)
+      else
+        return timeout
+      end
+    end
+    timeout
+  end
+
+  # dispatch a request like do_request, but queue +block+ for execution
+  # upon receiving a response.  It is the users' responsibility to ensure
+  # &block is executed in the correct order.  Trackers with multiple
+  # queryworkers are not guaranteed to return responses in the same
+  # order they were requested.
+  def pipeline_dispatch(cmd, args, &block) # :nodoc:
+    request = make_request(cmd, args)
+    timeout = @timeout
 
-  def do_request(cmd, args)
-    response = nil
-    request = make_request cmd, args
     @mutex.synchronize do
+      io = socket
+      timeout = pipeline_drain_unlocked(io, timeout)
+
+      # send the request out...
       begin
+        io.timed_write(request, timeout)
+        @pending << [ request, block ]
+      rescue SystemCallError, MogileFS::RequestTruncatedError => err
+        @dead[@active_host] = [ Time.now, err ]
+        shutdown_unlocked(@pending[0])
         io = socket
-        begin
-          bytes_sent = io.write request
-          bytes_sent == request.size or
-            raise MogileFS::RequestTruncatedError,
-               "request truncated (sent #{bytes_sent} expected #{request.size})"
-        rescue SystemCallError
-          raise MogileFS::UnreachableBackendError
-        end
+        retry
+      end
 
-        readable?(io)
-        response = io.gets(RS) and return parse_response(response)
-      ensure
-        # we DO NOT want the response we timed out waiting for, to crop up later
-        # on, on the same socket, intersperesed with a subsequent request!
-        # we close the socket if it times out like this
-        response or shutdown_unlocked
+      @pending.size
+    end
+  end
+
+  def pipeline_wait(count = nil) # :nodoc:
+    @mutex.synchronize do
+      io = socket
+      count ||= @pending.size
+      @pending.size < count and
+        raise MogileFS::Error,
+              "pending=#{@pending.size} < expected=#{count} failed"
+      begin
+        count.times { pipeline_gets_unlocked(io, @timeout) }
+      rescue
+        shutdown_unlocked(true)
       end
+    end
+  end
+
+  # Performs the +cmd+ request with +args+.
+  def do_request(cmd, args, idempotent = false)
+    request = make_request cmd, args
+    @mutex.synchronize do
+      begin
+        io = dispatch_unlocked(request)
+        line = io.timed_gets(@timeout) and return parse_response(line)
+
+        idempotent or
+          raise EOFError, "end of file reached after: #{request.inspect}"
+        # fall through to retry in loop
+      rescue SystemCallError,
+             MogileFS::UnreadableSocketError,
+             MogileFS::InvalidResponseError, # truncated response
+             MogileFS::Timeout
+        # we got a successful timed_write, but not a timed_gets
+        retry if idempotent
+        shutdown_unlocked(true)
+      rescue
+        # we DO NOT want the response we timed out waiting for, to crop up later
+        # on, on the same socket, intersperesed with a subsequent request!  we
+        # close the socket if there's any error.
+        shutdown_unlocked(true)
+      end while idempotent
     end # @mutex.synchronize
   end
 
-  ##
   # Makes a new request string for +cmd+ and +args+.
-
   def make_request(cmd, args)
     "#{cmd} #{url_encode args}\r\n"
   end
@@ -200,102 +292,120 @@ class MogileFS::Backend
     BACKEND_ERRORS[err_snake] || self.class.add_error(err_snake)
   end
 
-  ##
   # Turns the +line+ response from the server into a Hash of options, an
   # error, or raises, as appropriate.
-
-  def parse_response(line)
+  def parse_response(line, request = nil)
     if line =~ /^ERR\s+(\w+)\s*([^\r\n]*)/
       @lasterr = $1
       @lasterrstr = $2 ? url_unescape($2) : nil
-      raise error(@lasterr), @lasterrstr
+      if request
+        request = " request=#{request.strip}"
+        @lasterrstr = @lasterrstr ? (@lasterrstr << request) : request
+        return error(@lasterr).new(@lasterrstr)
+      end
+      raise error(@lasterr).new(@lasterrstr)
     end
 
-    return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)/
+    return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)\r\n\z/
 
     raise MogileFS::InvalidResponseError,
           "Invalid response from server: #{line.inspect}"
   end
 
-  ##
-  # Raises if the socket does not become readable in +@timeout+ seconds.
-
-  def readable?(io = @socket)
-    timeleft = @timeout
-    peer = nil
-    loop do
-      t0 = Time.now
-      found = IO.select([io], nil, nil, timeleft)
-      return true if found && found[0]
-      timeleft -= (Time.now - t0)
-      timeleft >= 0 and next
-      peer = io ? "#{io.mogilefs_peername} " : nil
+  # this command is special since the cache is per-tracker, so we connect
+  # to all backends and not just one
+  def clear_cache(types = %w(all))
+    opts = {}
+    types.each { |type| opts[type] = 1 }
 
-      raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
+    sockets = @hosts.map do |host|
+      MogileFS::Socket.start(*(host.split(/:/))) rescue nil
     end
-    false
+    sockets.compact!
+
+    wpending = sockets
+    rpending = []
+    request = make_request("clear_cache", opts)
+    while wpending[0] || rpending[0]
+      r = IO.select(rpending, wpending, nil, @timeout) or return
+      rpending -= r[0]
+      wpending -= r[1]
+      r[0].each { |io| io.timed_gets(0) rescue nil }
+      r[1].each do |io|
+        begin
+          io.timed_write(request, 0)
+          rpending << io
+        rescue
+        end
+      end
+    end
+    nil
+    ensure
+      sockets.each { |io| io.close }
   end
 
-  ##
   # Returns a socket connected to a MogileFS tracker.
-
   def socket
     return @socket if @socket and not @socket.closed?
 
     now = Time.now
 
-    @hosts.sort_by { rand(3) - 1 }.each do |host|
-      next if @dead.include? host and @dead[host] > now - 5
+    @hosts.shuffle.each do |host|
+      next if @dead.include?(host) and @dead[host][0] > now - 5
 
       begin
-        @socket = Socket.mogilefs_new(*(host.split(/:/) << @timeout))
-      rescue SystemCallError, MogileFS::Timeout
-        @dead[host] = now
+        addr, port = host.split(/:/)
+        @socket = MogileFS::Socket.tcp(addr, port, @timeout)
+        @active_host = host
+      rescue SystemCallError, MogileFS::Timeout => err
+        @dead[host] = [ now, err ]
         next
       end
 
       return @socket
     end
 
-    raise MogileFS::UnreachableBackendError
+    errors = @dead.map { |host,(_,e)| "#{host} - #{e.message} (#{e.class})" }
+    raise MogileFS::UnreachableBackendError,
+          "couldn't connect to any tracker: #{errors.join(', ')}"
   end
 
-  ##
   # Turns a url params string into a Hash.
-
-  def url_decode(str)
-    Hash[*(str.split(/&/).map { |pair|
-      pair.split(/=/, 2).map { |x| url_unescape(x) }
-    } ).flatten]
+  def url_decode(str) # :nodoc:
+    rv = {}
+    str.split(/&/).each do |pair|
+      k, v = pair.split(/=/, 2).map! { |x| url_unescape(x) }
+      rv[k.freeze] = v
+    end
+    rv
   end
 
-  ##
-  # Turns a Hash (or Array of pairs) into a url params string.
+  # :stopdoc:
+  # TODO: see if we can use existing URL-escape/unescaping routines
+  # in the Ruby standard library, Perl MogileFS seems to NIH these
+  #  routines, too
+  # :startdoc:
 
-  def url_encode(params)
+  # Turns a Hash (or Array of pairs) into a url params string.
+  def url_encode(params) # :nodoc:
     params.map do |k,v|
       "#{url_escape k.to_s}=#{url_escape v.to_s}"
     end.join("&")
   end
 
-  ##
   # Escapes naughty URL characters.
   if ''.respond_to?(:ord) # Ruby 1.9
-    def url_escape(str)
+    def url_escape(str) # :nodoc:
       str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1.ord }.tr(' ', '+')
     end
   else # Ruby 1.8
-    def url_escape(str)
+    def url_escape(str) # :nodoc:
       str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
     end
   end
 
-  ##
   # Unescapes naughty URL characters.
-
-  def url_unescape(str)
-    str.gsub(/%([a-f0-9][a-f0-9])/i) { [$1.to_i(16)].pack 'C' }.tr('+', ' ')
+  def url_unescape(str) # :nodoc:
+    str.tr('+', ' ').gsub(/%([a-f0-9][a-f0-9])/i) { [$1.to_i(16)].pack 'C' }
   end
-
 end
-

data/lib/mogilefs/bigfile.rb

@@ -1,107 +1,91 @@
 # -*- encoding: binary -*-
-require 'zlib'
-require 'digest/md5'
 require 'uri'
-require 'mogilefs/util'
+
+# Used for reading deprecated "bigfile" objects generated by the deprecated
+# mogtool(1) utility.  This is for reading legacy data and not recommended for
+# new projects.  MogileFS itself is capable of storing standalone objects
+# of arbitrary length (as long as the underlying database and underlying
+# filesystem on the DAV devices accept them).
 
 module MogileFS::Bigfile
-  GZIP_HEADER = "\x1f\x8b".freeze # mogtool(1) has this
   # VALID_TYPES = %w(file tarball partition).map { |x| x.freeze }.freeze
 
   # returns a big_info hash if successful
   def bigfile_stat(key)
-    parse_info(get_file_data(key))
+    bigfile_parse_info(get_file_data(key))
   end
 
   # returns total bytes written and the big_info hash if successful, raises an
-  # exception if not wr_io is expected to be an IO-like object capable of
-  # receiving the syswrite method.
+  # exception if not.  wr_io is expected to be an IO-like object capable of
+  # receiving the write method.
   def bigfile_write(key, wr_io, opts = { :verify => false })
     info = bigfile_stat(key)
-    zi = nil
-    md5 = opts[:verify] ? Digest::MD5.new : nil
     total = 0
+    t = @get_file_data_timeout
 
     # we only decode raw zlib deflated streams that mogtool (unfortunately)
     # generates.  tarballs and gzip(1) are up to to the application to decrypt.
-    filter = Proc.new do |buf|
-      if zi == nil
-        if info[:compressed] && info[:type] == 'file' &&
-             buf.length >= 2 && buf[0,2] != GZIP_HEADER
-          zi = Zlib::Inflate.new
-
-          # mogtool(1) seems to have a bug that causes it to generate bogus
-          # MD5s if zlib deflate is used.  Don't trust those MD5s for now...
-          md5 = nil
-        else
-          zi = false
-        end
-      end
-      buf ||= ''
-      if zi
-        zi.inflate(buf)
-      else
-        md5 << buf
-        buf
-      end
-    end if (info[:compressed] || md5)
+    if info[:compressed] || opts[:verify]
+      wr_io = MogileFS::Bigfile::Filter.new(wr_io, info, opts)
+    end
 
     info[:parts].each_with_index do |part,part_nr|
       next if part_nr == 0 # info[:parts][0] is always empty
-      uris = verify_uris(part[:paths].map { |path| URI.parse(path) })
-      if uris.empty?
+
+      begin
+        sock = MogileFS::HTTPReader.first(part[:paths], t)
+      rescue
         # part[:paths] may not be valid anymore due to rebalancing, however we
         # can get_keys on key,<part_nr> and retry paths if all paths fail
-        part[:paths] = get_paths("#{key.gsub(/^big_info:/, '')},#{part_nr}")
-        uris = verify_uris(part[:paths].map { |path| URI.parse(path) })
-        raise MogileFS::Backend::NoDevices if uris.empty?
+        part_key = "#{key.sub(/^_big_info:/, '')},#{part_nr}"
+        paths = get_paths(part_key)
+        paths.empty? and
+          raise MogileFS::Backend::NoDevices,
+                "no device for key=#{part_key.inspect}", []
+        sock = MogileFS::HTTPReader.first(paths, t)
       end
 
-      sock = http_read_sock(uris[0])
-      md5.reset if md5
-      w = sysrwloop(sock, wr_io, filter)
-
-      if md5 && md5.hexdigest != part[:md5]
-        raise MogileFS::ChecksumMismatchError, "#{md5} != #{part[:md5]}"
+      begin
+        w = MogileFS.io.copy_stream(sock, wr_io)
+      ensure
+        sock.close
       end
+
+      wr_io.respond_to?(:md5_check!) and wr_io.md5_check!(part[:md5])
       total += w
     end
-
-    syswrite_full(wr_io, zi.finish) if zi
+    wr_io.flush
+    total += wr_io.flushed_bytes if wr_io.respond_to?(:flushed_bytes)
 
     [ total, info ]
   end
 
-  private
-
-    include MogileFS::Util
-
-    ##
-    # parses the contents of a _big_info: string or IO object
-    def parse_info(info = '')
-      rv = { :parts => [] }
-      info.each_line do |line|
-        line.chomp!
-        case line
-        when /^(des|type|filename)\s+(.+)$/
-          rv[$1.to_sym] = $2
-        when /^compressed\s+([01])$/
-          rv[:compressed] = ($1 == '1')
-        when /^(chunks|size)\s+(\d+)$/
-          rv[$1.to_sym] = $2.to_i
-        when /^part\s+(\d+)\s+bytes=(\d+)\s+md5=(.+)\s+paths:\s+(.+)$/
-          rv[:parts][$1.to_i] = {
-            :bytes => $2.to_i,
-            :md5 => $3.downcase,
-            :paths => $4.split(/\s*,\s*/),
-          }
-        end
+  ##
+  # parses the contents of a _big_info: string or IO object
+  def bigfile_parse_info(info) # :nodoc:
+    rv = { :parts => [] }
+    info.each_line do |line|
+      line.chomp!
+      case line
+      when /^(des|type|filename)\s+(.+)$/
+        rv[$1.to_sym] = $2
+      when /^compressed\s+([01])$/
+        rv[:compressed] = ($1 == '1')
+      when /^(chunks|size)\s+(\d+)$/
+        rv[$1.to_sym] = $2.to_i
+      when /^part\s+(\d+)\s+bytes=(\d+)\s+md5=(.+)\s+paths:\s+(.+)$/
+        rv[:parts][$1.to_i] = {
+          :bytes => $2.to_i,
+          :md5 => $3.downcase,
+          :paths => $4.split(/\s*,\s*/),
+        }
       end
-
-      rv
     end
 
-end # module MogileFS::Bigfile
+    rv
+  end
+end
+require "mogilefs/bigfile/filter"
 
 __END__
 # Copied from mogtool: