mogilefs-client 2.2.0 → 3.0.0.rc1

data/lib/mogilefs/bigfile/filter.rb

@@ -0,0 +1,58 @@
+# -*- encoding: binary -*-
+require 'zlib'
+require 'digest/md5'
+
+# Filter class to wrap IO objects and uncompress DEFLATE'd files
+#
+# This is used for reading "bigfile" objects generated by the
+# (deprecated) mogtool(1)
+class MogileFS::Bigfile::Filter
+  GZIP_HEADER = "\x1f\x8b"
+  INFLATABLE_TYPES = { "file" => true }
+  attr_reader :flushed_bytes
+
+  def initialize(io, info, opts)
+    @io = io
+    @info = info
+    @md5 = opts[:verify] ? Digest::MD5.new : nil
+    @zi = nil
+    @flushed_bytes = 0
+  end
+
+  def md5_check!(expect)
+    return unless @md5
+    current = @md5.hexdigest
+    current == expect or
+      raise MogileFS::ChecksumMismatchError, "#{current} != #{expect}"
+    @md5.reset
+  end
+
+  def flush
+    @flushed_bytes = @io.write(@zi.finish) if @zi
+    @io.flush
+  end
+
+  def write(buf)
+    if nil == @zi
+      if @info[:compressed] &&
+         INFLATABLE_TYPES.include?(@info[:type]) &&
+         buf.bytesize >= 2 &&
+         buf[0,2] != GZIP_HEADER
+
+        @zi = Zlib::Inflate.new
+
+        # mogtool(1) seems to have a bug that causes it to generate bogus
+        # MD5s if zlib deflate is used.  Don't trust those MD5s for now...
+        @md5 = nil
+      else
+        @zi = false
+      end
+    end
+    if @zi
+      buf = @zi.inflate(buf)
+    else
+      @md5 << buf
+    end
+    @io.write(buf)
+  end
+end

data/lib/mogilefs/chunker.rb

@@ -0,0 +1,30 @@
+# -*- encoding: binary -*-
+require "digest/md5"
+class MogileFS::Chunker
+  CRLF = "\r\n"
+  attr_reader :io
+
+  def initialize(io, md5)
+    @io = io
+    @md5 = md5 ? Digest::MD5.new : nil
+  end
+
+  def write(buf)
+    rv = buf.bytesize
+    @io.write("#{rv.to_s(16)}\r\n")
+    @io.write(buf)
+    @md5.update(buf) if @md5
+    @io.write(CRLF)
+    rv
+  end
+
+  def flush
+    if @md5
+      content_md5 = [ @md5.digest ].pack('m').strip
+      warn "Content-MD5: #{content_md5}\r\n" if $DEBUG
+      @io.write("0\r\nContent-MD5: #{content_md5}\r\n\r\n")
+    else
+      @io.write("0\r\n\r\n")
+    end
+  end
+end

data/lib/mogilefs/client.rb

@@ -1,11 +1,9 @@
 # -*- encoding: binary -*-
-require 'mogilefs/backend'
 
 ##
 # MogileFS::Client is the MogileFS client base class.  Concrete clients like
 # MogileFS::MogileFS and MogileFS::Admin are implemented atop this one to do
 # real work.
-
 class MogileFS::Client
 
   ##

data/lib/mogilefs/copy_stream.rb

@@ -0,0 +1,30 @@
+# -*- encoding: binary -*-
+
+# internal compatibility class for older Rubies
+module MogileFS::CopyStream # :nodoc:
+  @r_args = IO::RDONLY | IO::NOCTTY
+  @w_args = [ IO::WRONLY|IO::CREAT|IO::NOCTTY|IO::TRUNC, 0600 ]
+  def self.copy_stream(src, dst)
+    src_io = src.respond_to?(:to_str) ? File.open(src, @r_args) : src
+    dst_io = dst.respond_to?(:to_str) ? File.open(dst, *@w_args) : dst
+    buf = ""
+    written = 0
+    if src_io.respond_to?(:readpartial)
+      begin
+        src_io.readpartial(0x4000, buf)
+        written += dst_io.write(buf)
+      rescue EOFError
+        break
+      end while true
+    else
+      while src_io.read(0x4000, buf)
+        written += dst_io.write(buf)
+      end
+    end
+    dst_io.flush if dst_io.respond_to?(:flush)
+    written
+    ensure
+      src_io.close if src.respond_to?(:to_str)
+      dst_io.close if dst.respond_to?(:to_str)
+  end
+end

data/lib/mogilefs/http_file.rb

@@ -0,0 +1,175 @@
+# -*- encoding: binary -*-
+# here are internal implementation details, do not use them in your code
+require 'stringio'
+require 'uri'
+require 'mogilefs/chunker'
+
+##
+# HTTPFile wraps up the new file operations for storing files onto an HTTP
+# storage node.
+#
+# You really don't want to create an HTTPFile by hand.  Instead you want to
+# create a new file using MogileFS::MogileFS.new_file.
+#
+class MogileFS::HTTPFile < StringIO
+  class EmptyResponseError < MogileFS::Error; end
+  class BadResponseError < MogileFS::Error; end
+  class UnparseableResponseError < MogileFS::Error; end
+  class NoStorageNodesError < MogileFS::Error
+    def message; 'Unable to open socket to storage node'; end
+  end
+  class NonRetryableError < MogileFS::Error; end
+
+  # :stopdoc:
+  MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
+  class << self
+    attr_accessor :response_timeout_cb
+  end
+
+  @response_timeout_cb = lambda do |elapsed_time, bytes_uploaded|
+    mbytes_uploaded = bytes_uploaded / (1024.0 * 1024.0)
+    # assumes worst case is 10M/s on the remote storage disk
+    t = mbytes_uploaded * 10 + elapsed_time
+    t < 5 ? 5 : t
+  end
+  # :startdoc:
+
+  ##
+  # The URI this file will be stored to.
+
+  attr_reader :uri
+
+  attr_reader :devid
+
+  ##
+  # The big_io name in case we have file > 256M
+
+  attr_accessor :big_io
+
+  attr_accessor :streaming_io
+
+  ##
+  # Creates a new HTTPFile with MogileFS-specific data.  Use
+  # MogileFS::MogileFS#new_file instead of this method.
+
+  def initialize(dests, content_length)
+    super ""
+    @streaming_io = @big_io = @uri = @devid = @active = nil
+    @dests = dests
+  end
+
+  def request_put(sock, uri, file_size, input = nil)
+    host_with_port = "#{uri.host}:#{uri.port}"
+    md5 = false
+    if MD5_TRAILER_NODES[host_with_port]
+      file_size = nil
+      md5 = true
+    end
+
+    if file_size
+      sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \
+                 "Content-Length: #{file_size}\r\n\r\n")
+      input ? MogileFS.io.copy_stream(@active = input, sock) : yield(sock)
+    else
+      trailers = md5 ? "Trailer: Content-MD5\r\n" : ""
+      sock.write("PUT #{uri.request_uri} HTTP/1.1\r\n" \
+                 "Host: #{host_with_port}\r\n#{trailers}" \
+                 "Transfer-Encoding: chunked\r\n\r\n")
+      tmp = MogileFS::Chunker.new(sock, md5)
+      rv = input ? MogileFS.io.copy_stream(@active = input, tmp) : yield(tmp)
+      tmp.flush
+      rv
+    end
+  end
+
+  def put_streaming_io(sock, uri) # unlikely to be used
+    file_size = @streaming_io.length
+    written = 0
+    request_put(sock, uri, file_size) do |wr|
+      @streaming_io.call(Proc.new do |data_to_write|
+        written += wr.write(data_to_write)
+      end)
+    end
+    file_size ? file_size : written
+  end
+
+  def rewind_or_raise!(uri, err)
+    @active.rewind if @active
+    rescue => e
+      msg = "#{uri} failed with #{err.message} (#{err.class}) and " \
+            "retrying is impossible as rewind on " \
+            "#{@active.inspect} failed with: #{e.message} (#{e.class})"
+      raise NonRetryableError, msg, e.backtrace
+  end
+
+  ##
+  # Writes an HTTP PUT request to +sock+ to upload the file and
+  # returns file size if the socket finished writing
+  def upload(devid, uri) # :nodoc:
+    start = Time.now
+    sock = MogileFS::Socket.tcp(uri.host, uri.port)
+    file_size = length
+
+    if @streaming_io
+      file_size = put_streaming_io(sock, uri)
+    elsif @big_io
+      if String === @big_io || @big_io.respond_to?(:to_path)
+        file = File.open(@big_io)
+        stat = file.stat
+        file_size = request_put(sock, uri, stat.file? ? stat.size : nil, file)
+      else
+        size = nil
+        if @big_io.respond_to?(:stat)
+          stat = @big_io.stat
+          size = stat.size if stat.file?
+        elsif @big_io.respond_to?(:size)
+          size = @big_io.size
+        end
+        file_size = request_put(sock, uri, size, @big_io)
+      end
+    else
+      rewind
+      request_put(sock, uri, file_size, self)
+    end
+
+    tout = self.class.response_timeout_cb.call(Time.now - start, file_size)
+
+    case line = sock.timed_read(23, "", tout)
+    when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
+      file_size
+    when nil
+      raise EmptyResponseError, 'Unable to read response line from server'
+    when %r{^HTTP/\d\.\d\s+(\d+)}
+      raise BadResponseError, "HTTP response status from upload: #$1"
+    else
+      raise UnparseableResponseError,
+            "Response line not understood: #{line.inspect}"
+    end
+    rescue => err
+      rewind_or_raise!(uri, err)
+      raise
+    ensure
+      file.close if file
+      sock.close if sock
+  end
+
+  def commit
+    errors = nil
+    @dests.each do |devid, path|
+      begin
+        uri = URI.parse(path)
+        bytes_uploaded = upload(devid, uri)
+        @devid, @uri = devid, uri
+        return bytes_uploaded
+      rescue NonRetryableError
+        raise
+      rescue => e
+        errors ||= []
+        errors << "#{path} - #{e.message} (#{e.class})"
+      end
+    end
+
+    raise NoStorageNodesError,
+          "all paths failed with PUT: #{errors.join(', ')}", []
+  end
+end

data/lib/mogilefs/http_reader.rb

@@ -0,0 +1,79 @@
+# -*- encoding: binary -*-
+# internal implementation details here, do not rely on them in your code
+
+# This class is needed because Net::HTTP streaming is still inefficient
+# for reading huge response bodies over fast LANs.
+class MogileFS::HTTPReader < MogileFS::Socket
+  attr_accessor :content_length, :uri
+
+  # backwards compat, if anybody cares
+  alias mogilefs_size content_length # :nodoc:
+
+  # this may OOM your system on large files
+  def to_s
+    buf = ""
+    read(@content_length, buf)
+    return buf if buf.size == @content_length
+
+    raise MogileFS::SizeMismatchError,
+          "read=#{buf.size} bytes, expected=#@content_length from #@uri", []
+  end
+
+  def stream_to(dest)
+    rv = MogileFS.io.copy_stream(self, dest)
+    return rv if rv == @content_length
+    raise MogileFS::SizeMismatchError,
+          "read=#{rv} bytes, expected=#@content_length from #@uri", []
+  end
+
+  def self.first(paths, timeout, count = nil, offset = nil)
+    errors = nil
+    if offset || count
+      offset ||= 0
+      range_end = count ? offset + count - 1 : ""
+      range = "Range: bytes=#{offset}-#{range_end}\r\n"
+    end
+
+    paths.each do |path|
+      begin
+        sock = try(path, timeout, range) and return sock
+      rescue => e
+        errors ||= []
+        errors << "#{path} - #{e.message} (#{e.class})"
+      end
+    end
+    raise MogileFS::Error,
+          "all paths failed with GET: #{errors.join(', ')}", []
+  end
+
+  # given a path, this returns a readable socket with ready data from the
+  # body of the response.
+  def self.try(path, timeout, range) # :nodoc:
+    uri = URI.parse(path)
+    sock = tcp(uri.host, uri.port, timeout)
+    buf = "GET #{uri.request_uri} HTTP/1.0\r\n#{range}\r\n" # no chunking
+    sock.timed_write(buf, timeout)
+
+    sock.timed_peek(2048, buf, timeout) or
+      raise MogileFS::InvalidResponseError, "EOF while reading header", []
+
+    head, _ = buf.split(/\r\n\r\n/, 2)
+
+    # we're dealing with a seriously slow/stupid HTTP server if we can't
+    # get the header in a single recv(2) syscall.
+    if ((range && head =~ %r{\AHTTP/\d+\.\d+\s+206\s*}) ||
+        (!range && head =~ %r{\AHTTP/\d+\.\d+\s+200\s*})) &&
+       head =~ %r{^Content-Length:\s*(\d+)}i
+      sock.content_length = $1.to_i
+      sock.uri = uri
+      sock.timed_read(head.bytesize + 4, buf, 0)
+      return sock
+    end
+    msg = range ? "Expected 206 w/#{range.strip}: " : "header="
+    msg << head.inspect
+    raise MogileFS::InvalidResponseError, msg, []
+  rescue
+    sock.close if sock
+    raise
+  end
+end

data/lib/mogilefs/mogilefs.rb

@@ -1,29 +1,44 @@
 # -*- encoding: binary -*-
-require 'mogilefs/client'
-require 'mogilefs/util'
-
-##
-# MogileFS File manipulation client.
 
+# \MogileFS file manipulation client.
+#
+#   Create a new instance that will communicate with these trackers:
+#   hosts = %w[192.168.1.69:6001 192.168.1.70:6001]
+#   mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts)
+#
+#   # Stores "A bunch of text to store" into 'some_key' with a class of 'text'.
+#   mg.store_content('some_key', 'text', "A bunch of text to store")
+#
+#   # Retrieve data from 'some_key' as a string
+#   data = mg.get_file_data('some_key')
+#
+#   # Store the contents of 'image.jpeg' into the key 'my_image' with a
+#   # class of 'image'.
+#   mg.store_file('my_image', 'image', 'image.jpeg')
+#
+#   # Store the contents of 'image.jpeg' into the key 'my_image' with a
+#   # class of 'image' using an open IO object.
+#   File.open('image.jpeg') { |fp| mg.store_file('my_image', 'image', fp) }
+#
+#   # Retrieve the contents of 'my_image' into '/path/to/huge_file'
+#   # without slurping the entire contents into memory:
+#   mg.get_file_data('my_image', '/path/to/huge_file')
+#
+#   # Remove the key 'my_image' and 'some_key'.
+#   mg.delete('my_image')
+#   mg.delete('some_key')
+#
 class MogileFS::MogileFS < MogileFS::Client
-
-  include MogileFS::Util
   include MogileFS::Bigfile
 
-  ##
   # The domain of keys for this MogileFS client.
+  attr_accessor :domain
 
-  attr_reader :domain
-
-  ##
   # The timeout for get_file_data.  Defaults to five seconds.
-
   attr_accessor :get_file_data_timeout
 
-  ##
   # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
   # :domain specifying the domain of this client.
-
   def initialize(args = {})
     @domain = args[:domain]
 
@@ -38,70 +53,86 @@ class MogileFS::MogileFS < MogileFS::Client
     end
   end
 
-  ##
-  # Enumerates keys starting with +key+.
-
-  def each_key(prefix)
+  # Enumerates keys, limited by optional +prefix+
+  def each_key(prefix = "", &block)
     after = nil
-
-    keys, after = list_keys prefix
-
-    until keys.nil? or keys.empty? do
-      keys.each { |k| yield k }
-      keys, after = list_keys prefix, after
-    end
-
+    begin
+      keys, after = list_keys(prefix, after, 1000, &block)
+    end while keys && keys[0]
     nil
   end
 
-  ##
-  # Retrieves the contents of +key+.
-
-  def get_file_data(key, &block)
-    paths = get_paths(key) or return nil
-    paths.each do |path|
-      begin
-        sock = http_read_sock(URI.parse(path))
-        begin
-          return yield(sock) if block_given?
-          return sysread_full(sock, sock.mogilefs_size, @get_file_data_timeout)
-        ensure
-          sock.close rescue nil
-        end
-      rescue MogileFS::Timeout, MogileFS::InvalidResponseError,
-             Errno::ECONNREFUSED, EOFError, SystemCallError
-      end
+  # Retrieves the contents of +key+.  If +dst+ is specified, +dst+
+  # should be an IO-like object capable of receiving the +write+ method
+  # or a path name.  +copy_length+ may be specified to limit the number of
+  # bytes to retrieve, and +src_offset+ can be specified to specified the
+  # start position of the copy.
+  def get_file_data(key, dst = nil, copy_length = nil, src_offset = nil)
+    paths = get_paths(key)
+    sock = MogileFS::HTTPReader.first(paths, @get_file_data_timeout,
+                                      copy_length, src_offset)
+    if dst
+      sock.stream_to(dst)
+    elsif block_given?
+      yield(sock)
+    else
+      sock.to_s
     end
-    nil
+    ensure
+      sock.close if sock && ! sock.closed?
   end
 
-  ##
-  # Get the paths for +key+.
+  # Get the paths (URLs as strings) for +key+.  If +args+ is specified,
+  # it may contain:
+  # - :noverify -> boolean, whether or not the tracker checks (default: true)
+  # - :pathcount -> a positive integer of URLs to retrieve (default: 2)
+  # - :zone -> "alt" or nil (default: nil)
+  #
+  # :noverify defaults to false because this client library is capable of
+  # verifying paths for readability itself.  It is also faster and more
+  # reliable to verify paths on the client.
+  def get_paths(key, *args)
+    opts = {
+      :domain => @domain,
+      :key => key,
+      :noverify => args[0],
+      :zone => args[1],
+    }
+    if Hash === args[0]
+      args = args[0]
+      opts[:noverify] = args[:noverify]
+      opts[:zone] = args[:zone]
+      pathcount = args[:pathcount] and opts[:pathcount] = pathcount.to_i
+    end
 
-  def get_paths(key, noverify = true, zone = nil)
-    opts = { :domain => @domain, :key => key,
-             :noverify => noverify ? 1 : 0, :zone => zone }
+    opts[:noverify] = false == opts[:noverify] ? 0 : 1
     @backend.respond_to?(:_get_paths) and return @backend._get_paths(opts)
     res = @backend.get_paths(opts)
-    (1..res['paths'].to_i).map { |i| res["path#{i}"] }.compact
+    (1..res['paths'].to_i).map { |i| res["path#{i}"] }
   end
 
-  ##
-  # Get the URIs for +key+.
+  # Returns +true+ if +key+ exists, +false+ if not
+  def exist?(key)
+    rv = nil
+    args = { :key => key, :domain => @domain }
+    @backend.pipeline_dispatch(:get_paths, args) { |x| rv = (Hash === x) }
+    @backend.pipeline_wait(1)
+    rv
+  end
 
-  def get_uris(key, noverify = true, zone = nil)
-    get_paths(key, noverify, zone).map { |path| URI.parse(path) }
+  # Get the URIs for +key+ (paths) as URI::HTTP objects
+  def get_uris(key, *args)
+    get_paths(key, *args).map! { |path| URI.parse(path) }
   end
 
-  ##
   # Creates a new file +key+ in +klass+.  +bytes+ is currently unused.
-  #
-  # The +block+ operates like File.open.
-
-  def new_file(key, klass = nil, bytes = 0, &block) # :yields: file
+  # Consider using store_file instead of this method for large files.
+  # This requires a block passed to it and operates like File.open.
+  # This atomically replaces existing data stored as +key+ when
+  def new_file(key, klass = nil, bytes = 0) # :yields: file
     raise MogileFS::ReadOnlyError if readonly?
     opts = { :domain => @domain, :key => key, :multi_dest => 1 }
-    opts[:class] = klass if klass
+    opts[:class] = klass if klass && klass != "default"
     res = @backend.create_open(opts)
 
     dests = if dev_count = res['dev_count'] # multi_dest succeeded
@@ -119,43 +150,38 @@ class MogileFS::MogileFS < MogileFS::Client
     end
 
     case (dests[0][1] rescue nil)
-    when nil, '' then
-      raise MogileFS::EmptyPathError
     when /^http:\/\// then
-      MogileFS::HTTPFile.open(self, res['fid'], klass, key,
-                              dests, bytes, &block)
+      http_file = MogileFS::HTTPFile.new(dests, bytes)
+      yield http_file
+      rv = http_file.commit
+      @backend.create_close(:fid => res['fid'],
+                            :devid => http_file.devid,
+                            :domain => @domain,
+                            :key => key,
+                            :path => http_file.uri.to_s,
+                            :size => rv)
+      rv
+    when nil, '' then
+      raise MogileFS::EmptyPathError,
+            "Empty path for mogile upload res=#{res.inspect}"
     else
       raise MogileFS::UnsupportedPathError,
             "paths '#{dests.inspect}' returned by backend is not supported"
     end
   end
 
-  ##
   # Copies the contents of +file+ into +key+ in class +klass+.  +file+ can be
-  # either a file name or an object that responds to #sysread.
-  # Returns size of +file+ stored
-
+  # either a path name (String or Pathname object) or an IO-like object that
+  # responds to #read or #readpartial.  Returns size of +file+ stored.
+  # This atomically replaces existing data stored as +key+
   def store_file(key, klass, file)
     raise MogileFS::ReadOnlyError if readonly?
 
-    new_file key, klass do |mfp|
-      if file.respond_to? :sysread then
-        sysrwloop(file, mfp)
-      else
-        size = File.size(file)
-        if size > 0x10000 # Bigass file, handle differently
-          mfp.big_io = file
-          size
-        else
-          File.open(file, "rb") { |fp| sysrwloop(fp, mfp) }
-        end
-      end
-    end
+    new_file(key, klass) { |mfp| mfp.big_io = file }
   end
 
-  ##
-  # Stores +content+ into +key+ in class +klass+.
-
+  # Stores +content+ into +key+ in class +klass+, where +content+ is a String
+  # This atomically replaces existing data stored as +key+
   def store_content(key, klass, content)
     raise MogileFS::ReadOnlyError if readonly?
 
@@ -166,29 +192,22 @@ class MogileFS::MogileFS < MogileFS::Client
         mfp << content
       end
     end
-
-    content.length
   end
 
-  ##
   # Removes +key+.
-
   def delete(key)
     raise MogileFS::ReadOnlyError if readonly?
 
     @backend.delete :domain => @domain, :key => key
+    true
   end
 
-  ##
-  # Sleeps +duration+.
-
-  def sleep(duration)
+  # Sleeps +duration+, only used for testing
+  def sleep(duration) # :nodoc:
     @backend.sleep :duration => duration
   end
 
-  ##
   # Renames a key +from+ to key +to+.
-
   def rename(from, to)
     raise MogileFS::ReadOnlyError if readonly?
 
@@ -196,80 +215,155 @@ class MogileFS::MogileFS < MogileFS::Client
     nil
   end
 
-  ##
   # Returns the size of +key+.
   def size(key)
     @backend.respond_to?(:_size) and return @backend._size(domain, key)
-    paths = get_paths(key) or return nil
-    paths_size(paths)
+    begin
+      file_info(key)["length"].to_i
+    rescue MogileFS::Backend::UnknownCommandError
+      paths_size(get_paths(key))
+    end
   end
 
-  def paths_size(paths)
-    paths.each do |path|
-      begin
-        return http_read_sock(URI.parse(path), "HEAD").mogilefs_size
-      rescue MogileFS::InvalidResponseError, MogileFS::Timeout,
-             Errno::ECONNREFUSED, EOFError, SystemCallError => err
-        next
-      end
-    end
-    nil
+  def paths_size(paths) # :nodoc:
+    require "mogilefs/paths_size"
+    MogileFS::PathsSize.call(paths)
   end
 
-  ##
-  # Lists keys starting with +prefix+ follwing +after+ up to +limit+.  If
+  # Lists keys starting with +prefix+ following +after+ up to +limit+.  If
   # +after+ is nil the list starts at the beginning.
-
-  def list_keys(prefix, after = nil, limit = 1000, &block)
-    if @backend.respond_to?(:_list_keys)
+  def list_keys(prefix = "", after = nil, limit = 1000, &block)
+    @backend.respond_to?(:_list_keys) and
       return @backend._list_keys(domain, prefix, after, limit, &block)
-    end
 
-    res = begin
-      @backend.list_keys(:domain => domain, :prefix => prefix,
-                         :after => after, :limit => limit)
+    begin
+      res = @backend.list_keys(:domain => domain, :prefix => prefix,
+                               :after => after, :limit => limit)
     rescue MogileFS::Backend::NoneMatchError
-      return nil
+      return
     end
 
     keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
-    if block_given?
-      # emulate the MogileFS::Mysql interface, slowly...
-      keys.each do |key|
-        paths = get_paths(key) or next
-        length = paths_size(paths) or next
-        yield key, length, paths.size
+    if block
+      if 1 == block.arity
+        keys.each { |key| block.call(key) }
+      else
+        list_keys_verbose(keys, block)
       end
     end
 
     [ keys, res['next_after'] ]
   end
 
-  protected
-
-    # given a URI, this returns a readable socket with ready data from the
-    # body of the response.
-    def http_read_sock(uri, http_method = "GET")
-      sock = Socket.mogilefs_new_request(uri.host, uri.port,
-                    "#{http_method} #{uri.request_uri} HTTP/1.0\r\n\r\n",
-                    @get_file_data_timeout)
-      buf = sock.recv_nonblock(4096, Socket::MSG_PEEK)
-      head, body = buf.split(/\r\n\r\n/, 2)
-
-      # we're dealing with a seriously slow/stupid HTTP server if we can't
-      # get the header in a single read(2) syscall.
-      if head =~ %r{\AHTTP/\d+\.\d+\s+200\s*} &&
-         head =~ %r{^Content-Length:\s*(\d+)}i
-        sock.mogilefs_size = $1.to_i
-        case http_method
-        when "HEAD" then sock.close
-        when "GET" then sock.recv(head.size + 4, 0)
-        end
-        return sock
+  def list_keys_verbose(keys, block) # :nodoc:
+    # emulate the MogileFS::Mysql interface, slowly...
+    ordered = keys.dup
+    ready = {}
+    on_file_info = lambda do |info|
+      Hash === info or raise info
+      file_info_cleanup(info)
+
+      # deal with trackers with multiple queryworkers responding out-of-order
+      ready[info["key"]] = info
+      while info = ready.delete(ordered[0])
+        block.call(ordered.shift, info["length"], info["devcount"])
+      end
+    end
+    opts = { :domain => @domain }
+    begin
+      keys.each do |key|
+        opts[:key] = key
+        @backend.pipeline_dispatch(:file_info, opts, &on_file_info)
       end
-      sock.close rescue nil
-      raise MogileFS::InvalidResponseError,
-            "#{http_method} on #{uri} returned: #{head.inspect}"
-    end # def http_read_sock
+      @backend.pipeline_wait
+    rescue MogileFS::Backend::UnknownCommandError # MogileFS < 2.45
+      @backend.shutdown # reset the socket
+      args = { :pathcount => 0x7fffffff }
+      keys.each do |key|
+        paths = get_paths(key, args)
+        block.call(key, paths_size(paths), paths.size)
+      end
+    rescue MogileFS::PipelineError, SystemCallError,
+           MogileFS::RequestTruncatedError,
+           MogileFS::UnreadableSocketError,
+           MogileFS::InvalidResponseError, # truncated response
+           MogileFS::Timeout
+      @backend.shutdown
+      keys = ordered - ready.keys
+      retry
+    rescue
+      @backend.shutdown
+      raise
+    end
+  end
+
+  # Return metadata about a file as a hash.
+  # Returns the domain, class, length, devcount, etc. as keys.
+  # Optionally, device ids (not paths) can be returned as
+  # well if :devices is specified and +true+.
+  #
+  # This should only be used for informational purposes, and not usually
+  # for dynamically serving files.
+  #
+  #   mg.file_info("bar")
+  #
+  # Returns:
+  #
+  #   {
+  #     "domain" => "foo",
+  #     "key" => "bar",
+  #     "class" => "default",
+  #     "devcount" => 2,
+  #     "length => 666
+  #   }
+  def file_info(key, args = nil)
+    opts = { :domain => @domain, :key => key }
+    args and devices = args[:devices] and opts[:devices] = devices ? 1 : 0
+    file_info_cleanup(@backend.file_info(opts))
+  end
+
+  def file_info_cleanup(rv) # :nodoc:
+    %w(fid length devcount).each { |f| rv[f] = rv[f].to_i }
+    devids = rv["devids"] and
+      rv["devids"] = devids.split(/,/).map! { |x| x.to_i }
+    rv
+  end
 
+  # Given an Integer +fid+ or String +key+ and domain, thorougly search
+  # the database for all occurences of a particular fid.
+  #
+  # Use this sparingly, this command hits the master database numerous
+  # times and is very expensive.  This is not for production use, only
+  # troubleshooting and debugging.
+  #
+  # Searches for fid=666:
+  #
+  #   client.file_debug(666)
+  #
+  # Search for key=foo using the default domain for this object:
+  #
+  #   client.file_debug("foo")
+  #
+  # Search for key=foo in domain="bar":
+  #
+  #   client.file_debug(:key => "foo", :domain => "bar")
+  #
+  def file_debug(args)
+    case args
+    when Integer then args = { "fid" => args }
+    when String then args = { "key" => args }
+    end
+    opts = { :domain => args[:domain] || @domain }.merge!(args)
+
+    rv = @backend.file_debug(opts)
+    rv.each do |k,v|
+      case k
+      when /_(?:classid|devcount|dmid|fid|length|
+            nexttry|fromdevid|failcount|flags|devid|type)\z/x
+        rv[k] = v.to_i
+      when /devids\z/
+        rv[k] = v.split(/,/).map! { |x| x.to_i }
+      end
+    end
+  end
 end