mogilefs-client 2.2.0 → 3.0.0.rc1

data/examples/mogstored_rack.rb

@@ -0,0 +1,189 @@
+# -*- encoding: binary -*-
+require 'tempfile'
+require 'digest/md5'
+require 'rack'
+
+# Rack application for handling HTTP PUT/DELETE/MKCOL operations needed
+# for a MogileFS storage server.  GET requests are handled by
+# Rack::File and Rack::Head _must_ be in the middleware stack for
+# mogilefsd fsck to work properly with keepalive.
+#
+# Usage in rackup config file (config.ru):
+#
+#    require "./mogstored_rack"
+#    use Rack::Head
+#    run MogstoredRack.new("/var/mogdata")
+class MogstoredRack
+  class ContentMD5 < Digest::MD5
+    def content_md5
+      [ digest ].pack("m").strip!
+    end
+  end
+
+  def initialize(root, opts = {})
+    @root = File.expand_path(root)
+    @rack_file = (opts[:app] || Rack::File.new(@root))
+    @fsync = !! opts[:fsync]
+    @creat_perms = opts[:creat_perms] || (~File.umask & 0666)
+    @mkdir_perms = opts[:mkdir_perms] || (~File.umask & 0777)
+    @reread_verify = !! opts[:reread_verify]
+  end
+
+  def call(env)
+    case env["REQUEST_METHOD"]
+    when "GET", "HEAD"
+      case env["PATH_INFO"]
+      when "/"
+        r(200, "") # MogileFS seems to need this...
+      else
+        @rack_file.call(env)
+      end
+    when "PUT"
+      put(env)
+    when "DELETE"
+      delete(env)
+    when "MKCOL"
+      mkcol(env)
+    else
+      r(405, "unsupported method", env)
+    end
+    rescue Errno::EPERM, Errno::EACCES => err
+      r(403, "#{err.message} (#{err.class})", env)
+    rescue => err
+      r(500, "#{err.message} (#{err.class})", env)
+  end
+
+  def mkcol(env)
+    path = server_path(env) or return r(400)
+    Dir.mkdir(path, @mkdir_perms)
+    r(204)
+    rescue Errno::EEXIST # succeed (204) on race condition
+      File.directory?(path) ? r(204) : r(409)
+  end
+
+  def delete(env)
+    path = server_path(env) or return r(400)
+    File.exist?(path) or return r(404)
+    File.directory?(path) ? Dir.rmdir(path) : File.unlink(path)
+    r(204)
+    rescue Errno::ENOENT # return 404 on race condition
+      File.exist?(path) ? r(500) : r(404)
+  end
+
+  def put(env)
+    path = server_path(env) or return r(400)
+    dir = File.dirname(path)
+    File.directory?(dir) or return r(403)
+
+    Tempfile.open([File.basename(path), ".tmp"], dir) do |tmp|
+      tmp = tmp.to_io # delegated method calls are slower
+      tmp.sync = true
+      tmp.binmode
+      buf = ""
+      received = put_loop(env["rack.input"], tmp, buf)
+      err = content_md5_fail?(env, received) and return err
+      if @reread_verify && err = reread_md5_fail?(env, tmp, received, buf)
+        return err
+      end
+      tmp.chmod(@creat_perms)
+      begin
+        File.link(tmp.path, path)
+      rescue Errno::EEXIST
+        err = rename_overwrite_fail?(tmp.path, path) and return err
+      end
+      fsync(dir, tmp) if @fsync
+      resp = r(201)
+      resp[1]["X-Received-Content-MD5"] = received
+      return resp
+    end
+  end
+
+  def put_loop(src, dst, buf)
+    md5 = ContentMD5.new
+    while src.read(0x4000, buf)
+      md5.update(buf)
+      dst.write(buf)
+    end
+    md5.content_md5
+  end
+
+  def server_path(env)
+    path = env['PATH_INFO'].squeeze('/')
+    path.split(%r{/}).include?("..") and return false
+    "#@root#{path}"
+  end
+
+  # returns a plain-text HTTP response
+  def r(code, msg = nil, env = nil)
+    if env && logger = env["rack.logger"]
+      logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \
+                  "#{code} #{msg.inspect}")
+    end
+    if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code)
+      [ code, {}, [] ]
+    else
+      msg ||= Rack::Utils::HTTP_STATUS_CODES[code] || ""
+      msg += "\n" if msg.size > 0
+      [ code,
+        { 'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s },
+        [ msg ] ]
+    end
+  end
+
+  # Tries to detect filesystem/disk corruption.
+  # Unfortunately, posix_fadvise(2)/IO#advise is only advisory and
+  # can't guarantee we're not just reading the data in the kernel
+  # page cache.
+  def reread_md5_fail?(env, tmp, received, buf)
+    # try to force a reread from the storage device, not cache
+    tmp.fsync
+    tmp.rewind
+    tmp.advise(:dontneed) rescue nil # only in Ruby 1.9.3 and only advisory
+
+    md5 = ContentMD5.new
+    while tmp.read(0x4000, buf)
+      md5.update(buf)
+    end
+    reread = md5.content_md5
+    reread == received and return false # success
+    r(500, "reread MD5 mismatch\n" \
+           "received: #{received}\n" \
+           "  reread: #{reread}", env)
+  end
+
+  # Tries to detect network corruption by verifying the client-supplied
+  # Content-MD5 is correct.  It's highly unlikely the MD5 can be corrupted
+  # in a way that also allows corrupt data to pass through.
+  #
+  # The Rainbows!/Unicorn HTTP servers will populate the HTTP_CONTENT_MD5
+  # field in +env+ after env["rack.input"] is fully-consumed.  Clients
+  # may also send Content-MD5 as a header and this will still work.
+  def content_md5_fail?(env, received)
+    expected = env["HTTP_CONTENT_MD5"] or return false
+    expected = expected.strip
+    expected == received and return false # success
+    r(400, "Content-MD5 mismatch\n" \
+           "expected: #{expected}\n" \
+           "received: #{received}", env)
+  end
+
+  def rename_overwrite_fail?(src, dst)
+    10.times do
+      begin
+        tmp_dst = "#{dst}.#{rand}"
+        File.link(src, tmp_dst)
+      rescue Errno::EEXIST
+        next
+      end
+      File.rename(tmp_dst, dst)
+      return false # success!
+    end
+    r(409)
+  end
+
+  # fsync each and every directory component above us on the same device
+  def fsync(dir, tmp)
+    tmp.fsync
+    File.open(dir) { |io| io.fsync }
+  end
+end

data/lib/mogilefs.rb

@@ -1,41 +1,80 @@
 # -*- encoding: binary -*-
-##
-# MogileFS is a Ruby client for Danga Interactive's open source distributed
-# filesystem.
 #
-# To read more about Danga's MogileFS: http://danga.com/mogilefs/
-
+# To read more about \MogileFS, go to http://mogilefs.org/
+#
+# Client usage information is available in MogileFS::MogileFS.
 module MogileFS
 
-  VERSION = '2.2.0'
+  # Standard error class for most MogileFS-specific errors
+  class Error < StandardError; end
 
-  ##
   # Raised when a socket remains unreadable for too long.
-
-  class Error < StandardError; end
   class UnreadableSocketError < Error; end
+
+  # Raised when a response is truncated while reading
+  # due to network/server # errors)
   class SizeMismatchError < Error; end
+
+  # Raised when checksum verification fails (only while reading deprecated
+  # "bigfiles" from the deprecated mogtool(1).
   class ChecksumMismatchError < RuntimeError; end
+
+  # Raised when a backend is in read-only mode
   class ReadOnlyError < Error
     def message; 'readonly mogilefs'; end
   end
-  class EmptyPathError < Error
-    def message; 'Empty path for mogile upload'; end
-  end
 
+  # Raised when an upload is impossible
+  class EmptyPathError < Error; end
+
+  # Raised when we are given an unsupported protocol to upload to.
+  # Currently, the \MogileFS (2.55) server only supports HTTP and
+  # this library is only capable of HTTP.
   class UnsupportedPathError < Error; end
+
+  # Raised when a request (HTTP or tracker) was truncated due to a network or
+  # server error.  It may be possible to retry idempotent requests from this.
   class RequestTruncatedError < Error; end
+
+  # Raised when a response from a server (HTTP or tracker) is not in a format
+  # that we expected (or truncated..
   class InvalidResponseError < Error; end
-  class UnreachableBackendError < Error
-    def message; "couldn't connect to mogilefsd backend"; end
+
+  # Raised when all known backends have failed.
+  class UnreachableBackendError < Error; end
+
+  # There was an error as a result of the use of the (experimental)
+  # pipelining code to the tracker backend
+  class PipelineError < Error; end
+
+  # :stopdoc:
+  class << self
+    # somebody could use IO::Splice from the "io_splice" RubyGem, too
+    # don't consider this a stable API, though...
+    attr_accessor :io
+  end
+
+  # IO.copy_stream was buggy in Ruby 1.9.2 and earlier
+  if RUBY_VERSION >= "1.9.3"
+    @io = IO
+  else
+    require "mogilefs/copy_stream"
+    @io = MogileFS::CopyStream
   end
 
+  # autoload rarely-used things:
+  autoload :Mysql, 'mogilefs/mysql'
+  autoload :Pool, 'mogilefs/pool'
+  autoload :Admin, 'mogilefs/admin'
+  # :startdoc:
 end
 
+require 'mogilefs/util'
+require 'mogilefs/socket'
 require 'mogilefs/backend'
-require 'mogilefs/httpfile'
+require 'mogilefs/http_file'
+require 'mogilefs/http_reader'
 require 'mogilefs/client'
 require 'mogilefs/bigfile'
 require 'mogilefs/mogilefs'
-require 'mogilefs/admin'
-
+require 'mogilefs/version' # generated by ./GIT-VERSION-GEN

data/lib/mogilefs/admin.rb

@@ -1,25 +1,22 @@
 # -*- encoding: binary -*-
-require 'mogilefs/client'
-
-##
-# A MogileFS Administration Client
 
+# \MogileFS administration client, this has little real-world usage
+# and is considered a work-in-progress
 class MogileFS::Admin < MogileFS::Client
 
   ##
-  # Enumerates fids using #list_fids.
+  # Enumerates fids using #list_fids.  Returns the number of valid fids
+  # processed
 
   def each_fid
-    low = 0
-    high = nil
-
-    max = get_stats('fids')['fids']['max']
-
-    0.step max, 100 do |high|
-      fids = list_fids low, high
+    low = -1
+    rv = 0
+    begin
+      fids = list_fids(low + 1)
       fids.each { |fid| yield fid }
-      low = high + 1
-    end
+      rv += fids.size
+    end while last = fids[-1] and low = last["fid"]
+    rv
   end
 
   ##
@@ -31,18 +28,18 @@ class MogileFS::Admin < MogileFS::Client
   # Returns:
   #
   #   [{"status"=>"alive",
-  #     "http_get_port"=>"",
+  #     "http_get_port"=>nil,
   #     "http_port"=>"",
   #     "hostid"=>"1",
   #     "hostip"=>"",
   #     "hostname"=>"rur-1",
-  #     "remoteroot"=>"/mnt/mogilefs/rur-1",
   #     "altip"=>"",
   #     "altmask"=>""}]
 
   def get_hosts(hostid = nil)
+    to_i = { "hostid" => true, "http_port" => true, "http_get_port" => true }
     clean('hosts', 'host',
-          @backend.get_hosts(hostid ? { :hostid => hostid } : {}))
+          @backend.get_hosts(hostid ? { :hostid => hostid } : {}), true, to_i)
   end
 
   ##
@@ -54,46 +51,60 @@ class MogileFS::Admin < MogileFS::Client
   # Returns:
   #
   #   [{"status"=>"alive",
-  #     "mb_asof"=>"",
-  #     "mb_free"=>"0",
-  #     "devid"=>"1",
-  #     "hostid"=>"1",
-  #     "mb_used"=>"",
-  #     "mb_total"=>""}]
+  #     "mb_asof"=>nil,
+  #     "mb_free"=>666000,
+  #     "devid"=>1,
+  #     "hostid"=>1,
+  #     "mb_used"=>666,
+  #     "mb_total"=>666666}]
 
   def get_devices(devid = nil)
-    clean('devices', 'dev',
-          @backend.get_devices(devid ? { :devid => devid } : {}))
+    to_i = {
+      "mb_asof" => true, "mb_free" => true,
+      "mb_used" => true, "mb_total" => true ,
+      "devid" => true, "weight" => true, "hostid" => true
+    }
+    rv = @backend.get_devices(devid ? { :devid => devid } : {})
+    rv = clean('devices', 'dev', rv, true, to_i)
+    rv.each do |row|
+      u = row["utilization"] and
+        row["utilization"] = nil == u ? nil : u.to_f
+    end
   end
 
   ##
-  # Returns an Array of fid Hashes from +from_fid+ to +to_fid+.
+  # Returns an Array of fid Hashes from +from_fid+, limited to +count+
   #
   #   admin.list_fids 0, 100
   #
   # Returns:
   #
-  #   [{"fid"=>"99",
+  #   [{"fid"=>99,
   #     "class"=>"normal",
   #     "domain"=>"test",
-  #     "devcount"=>"2",
-  #     "length"=>"4",
+  #     "devcount"=>2,
+  #     "length"=>4,
   #     "key"=>"file_key"},
-  #    {"fid"=>"82",
+  #    {"fid"=>82,
   #     "class"=>"normal",
-  #     "devcount"=>"2",
+  #     "devcount"=>2,
   #     "domain"=>"test",
-  #     "length"=>"9",
+  #     "length"=>9,
   #     "key"=>"new_new_key"}]
 
-  def list_fids(from_fid, to_fid)
+  def list_fids(from_fid, count = 100)
+    to_i = { "fid" => true, "devcount" => true, "length" => true }
+    # :to is now :count internally in mogilefsd
     clean('fid_count', 'fid_',
-          @backend.list_fids(:from => from_fid, :to => to_fid))
+          @backend.list_fids(:from => from_fid, :to => count), true, to_i)
   end
 
   ##
   # Returns a statistics structure representing the state of mogilefs.
   #
+  # *** This command no longer works with recent versions of MogileFS ***
+  # *** Use mogstats(1) from the MogileFS::Utils package on CPAN ***
+  #
   #   admin.get_stats
   #
   # Returns:
@@ -129,22 +140,47 @@ class MogileFS::Admin < MogileFS::Client
   end
 
   ##
-  # Returns the domains present in the mogilefs.
+  # Returns the domains and classes, and their policies present in the mogilefs.
   #
   #   admin.get_domains
   #
-  # Returns:
+  # Returns (on newer MogileFS servers):
+  #   {
+  #     "test" => {
+  #       "default" => {
+  #         "mindevcount" => 2,
+  #         "replpolicy" => "MultipleHosts()"
+  #       }
+  #     }
+  #   }
+  #
+  # Returns (on older MogileFS servers without replication policies):
   #
   #   {"test"=>{"normal"=>3, "default"=>2}}
 
   def get_domains
     res = @backend.get_domains
+    have_replpolicy = false
 
     domains = {}
+    to_i = { "mindevcount" => true }
     (1..res['domains'].to_i).each do |i|
-      domain = clean "domain#{i}classes", "domain#{i}class", res, false
-      domain = domain.map { |d| [d.values.first, d.values.last.to_i] }
-      domains[res["domain#{i}"]] = Hash[*domain.flatten]
+      domain = clean "domain#{i}classes", "domain#{i}class", res, false, to_i
+
+      tmp = domains[res["domain#{i}"].freeze] = {}
+      domain.each do |d|
+        tmp[d.delete("name").freeze] = d
+        have_replpolicy ||= d.include?("replpolicy")
+      end
+    end
+
+    # only for MogileFS 1.x?, maybe we can drop support for this...
+    unless have_replpolicy
+      domains.each do |namespace, class_data|
+        class_data.each do |class_name, data|
+          class_data[class_name] = data["mindevcount"]
+        end
+      end
     end
 
     domains
@@ -160,7 +196,8 @@ class MogileFS::Admin < MogileFS::Client
   end
 
   ##
-  # Deletes +domain+.  Returns true if successful, false if not.
+  # Deletes +domain+.  Returns true if successful, raises
+  # MogileFS::Backend::DomainNotFoundError if not
 
   def delete_domain(domain)
     raise MogileFS::ReadOnlyError if readonly?
@@ -168,24 +205,24 @@ class MogileFS::Admin < MogileFS::Client
   end
 
   ##
-  # Creates a new class in +domain+ named +klass+ with files replicated to
-  # +mindevcount+ devices.  Returns nil on failure.
+  # Creates a new class in +domain+ named +klass+ with +policy+ for
+  # replication.  Raises on failure.
 
-  def create_class(domain, klass, mindevcount)
-    modify_class(domain, klass, mindevcount, :create)
+  def create_class(domain, klass, policy)
+    modify_class(domain, klass, policy, :create)
   end
 
   ##
-  # Updates class +klass+ in +domain+ to be replicated to +mindevcount+
-  # devices.  Returns nil on failure.
+  # Updates class +klass+ in +domain+ with +policy+ for replication.
+  # Raises on failure.
 
-  def update_class(domain, klass, mindevcount)
-    modify_class(domain, klass, mindevcount, :update)
+  def update_class(domain, klass, policy)
+    modify_class(domain, klass, policy, :update)
   end
 
   ##
-  # Removes class +klass+ from +domain+.  Returns true if successful, false if
-  # not.
+  # Removes class +klass+ from +domain+.  Returns true if successful.
+  # Raises on failure
 
   def delete_class(domain, klass)
     ! @backend.delete_class(:domain => domain, :class => klass).nil?
@@ -226,18 +263,41 @@ class MogileFS::Admin < MogileFS::Client
     ! @backend.set_state(:host => host, :device => device, :state => state).nil?
   end
 
+  # reschedules all deferred replication, returns a hash with the number
+  # of files rescheduled:
+  #
+  #   admin.replicate_now => { "count" => 5 }
+  def replicate_now
+    rv = @backend.replicate_now
+    rv["count"] = rv["count"].to_i
+    rv
+  end
+
+  def clear_cache
+    @backend.clear_cache
+  end
+
   protected unless defined? $TESTING
 
   ##
   # Modifies +klass+ on +domain+ to store files on +mindevcount+ devices via
-  # +action+.  Returns the class name if successful, nil if not.
+  # +action+.  Returns the class name if successful, raises if not
 
-  def modify_class(domain, klass, mindevcount, action)
+  def modify_class(domain, klass, policy, action)
     raise MogileFS::ReadOnlyError if readonly?
-    res = @backend.send("#{action}_class", :domain => domain, :class => klass,
-                                          :mindevcount => mindevcount)
-
-    res ? res['class'] : nil
+    args = { :domain => domain, :class => klass }
+    case policy
+    when Integer
+      args[:mindevcount] = policy
+    when String
+      args[:replpolicy] = policy
+    when Hash
+      args.merge!(policy)
+    else
+      raise ArgumentError,
+           "policy=#{policy.inspect} not understood for #{action}_class"
+    end
+    @backend.__send__("#{action}_class", args)["class"]
   end
 
   ##
@@ -246,7 +306,7 @@ class MogileFS::Admin < MogileFS::Client
 
   def modify_host(host, args = {}, action = 'create')
     args[:host] = host
-    ! @backend.send("#{action}_host", args).nil?
+    ! @backend.__send__("#{action}_host", args).nil?
   end
 
   ##
@@ -278,13 +338,19 @@ class MogileFS::Admin < MogileFS::Client
   #     "altip"=>"",
   #     "altmask"=>""}]
 
-  def clean(count, prefix, res, underscore = true)
-    underscore = underscore ? '_' : ''
+  def clean(count, prefix, res, underscore = true, to_i = [])
+    empty = ""
+    underscore = underscore ? '_' : empty
+    keys = res.keys
     (1..res[count].to_i).map do |i|
-      dev = res.select { |k,_| k =~ /^#{prefix}#{i}#{underscore}/ }.map do |k,v|
-        [k.sub(/^#{prefix}#{i}#{underscore}/, ''), v]
+      re = /^#{prefix}#{i}#{underscore}/
+      row = {}
+      keys.grep(re).each do |k|
+        v = res[k]
+        k = k.sub(re, empty).freeze
+        row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
       end
-      Hash[*dev.flatten]
+      row
     end
   end