mogilefs-client 1.0.1

data/LICENSE

@@ -0,0 +1,32 @@
+Portions copyright 2004 David Heinemeier Hansson.
+
+All original code copyright 2005 Eric Hodel, The Robot Co-op.  All rights
+reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the names of the authors nor the names of their contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+4. Redistribution in Rails or any sub-projects of Rails is not allowed
+   until Rails runs without warnings with the ``-W2'' flag enabled.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/Manifest.txt

@@ -0,0 +1,18 @@
+LICENSE
+Manifest.txt
+README
+Rakefile
+lib/mogilefs.rb
+lib/mogilefs/admin.rb
+lib/mogilefs/backend.rb
+lib/mogilefs/client.rb
+lib/mogilefs/httpfile.rb
+lib/mogilefs/mogilefs.rb
+lib/mogilefs/nfsfile.rb
+lib/mogilefs/pool.rb
+test/setup.rb
+test/test_admin.rb
+test/test_backend.rb
+test/test_client.rb
+test/test_mogilefs.rb
+test/test_pool.rb

data/README

@@ -0,0 +1,9 @@
+This is a client for Danga's MogileFS file store.  For information on MogileFS
+see:
+
+http://danga.com/mogilefs/
+
+This client is only known to work in NFS mode.  HTTP mode is implemented but completely untested.  If you find a bug, please report it in an email to:
+
+eric@robotcoop.com.
+

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+
+$VERBOSE = nil
+
+spec = Gem::Specification.new do |s|
+  s.name = 'mogilefs-client'
+  s.version = '1.0.1'
+  s.summary = 'A Ruby MogileFS client'
+  s.description = 'A Ruby MogileFS client.  MogileFS is a distributed filesystem written by Danga Interactive.  This client supports NFS mode and has untested support for HTTP mode.'
+  s.author = 'Eric Hodel'
+  s.email = 'eric@robotcoop.com'
+
+  s.has_rdoc = true
+  s.files = File.read('Manifest.txt').split($/)
+  s.require_path = 'lib'
+end
+
+desc 'Run tests'
+task :default => [ :test ]
+
+Rake::TestTask.new('test') do |t|
+  t.libs << 'test'
+  t.pattern = 'test/test_*.rb'
+  t.verbose = true
+end
+
+desc 'Update Manifest.txt'
+task :update_manifest do
+  sh "find . -type f | sed -e 's%./%%' | egrep -v 'svn|swp|~' | egrep -v '^(doc|pkg)/' | sort > Manifest.txt"
+end
+
+desc 'Generate RDoc'
+Rake::RDocTask.new :rdoc do |rd|
+  rd.rdoc_dir = 'doc'
+  rd.rdoc_files.add 'lib', 'README', 'LICENSE'
+  rd.main = 'README'
+  rd.options << '-d' if `which dot` =~ /\/dot/
+end
+
+desc 'Build Gem'
+Rake::GemPackageTask.new spec do |pkg|
+  pkg.need_tar = true
+end
+
+desc 'Clean up'
+task :clean => [ :clobber_rdoc, :clobber_package ]
+
+desc 'Clean up'
+task :clobber => [ :clean ]
+
+# vim: syntax=Ruby
+

data/lib/mogilefs.rb

@@ -0,0 +1,24 @@
+##
+# MogileFS is a Ruby client for Danga Interactive's open source distributed
+# filesystem.
+#
+# To read more about Danga's MogileFS: http://danga.com/mogilefs/
+
+module MogileFS
+
+  ##
+  # Raised when a socket remains unreadable for too long.
+
+  class UnreadableSocketError < RuntimeError; end
+
+end
+
+require 'socket'
+
+require 'mogilefs/backend'
+require 'mogilefs/nfsfile'
+require 'mogilefs/httpfile'
+require 'mogilefs/client'
+require 'mogilefs/mogilefs'
+require 'mogilefs/admin'
+

data/lib/mogilefs/admin.rb

@@ -0,0 +1,274 @@
+require 'mogilefs/client'
+
+##
+# A MogileFS Administration Client
+
+class MogileFS::Admin < MogileFS::Client
+
+  ##
+  # Returns an Array of host status Hashes.  If +hostid+ is given only that
+  # host is returned.
+  #
+  #   admin.get_hosts 1
+  #
+  # Returns:
+  #
+  #   [{"status"=>"alive",
+  #     "http_get_port"=>"",
+  #     "http_port"=>"",
+  #     "hostid"=>"1",
+  #     "hostip"=>"",
+  #     "hostname"=>"rur-1",
+  #     "remoteroot"=>"/mnt/mogilefs/rur-1",
+  #     "altip"=>"",
+  #     "altmask"=>""}]
+
+  def get_hosts(hostid = nil)
+    args = hostid ? { :hostid => hostid } : {}
+    res = @backend.get_hosts args
+    return clean('hosts', 'host', res)
+  end
+
+  ##
+  # Returns an Array of device status Hashes.  If devid is given only that
+  # device is returned.
+  #
+  #   admin.get_devices 1
+  #
+  # Returns:
+  #
+  #   [{"status"=>"alive",
+  #     "mb_asof"=>"",
+  #     "mb_free"=>"0",
+  #     "devid"=>"1",
+  #     "hostid"=>"1",
+  #     "mb_used"=>"",
+  #     "mb_total"=>""}]
+
+  def get_devices(devid = nil)
+    args = devid ? { :devid => devid } : {}
+    res = @backend.get_devices args
+    return clean('devices', 'dev', res)
+  end
+
+  ##
+  # Returns an Array of fid Hashes from +from_fid+ to +to_fid+.
+  #
+  #   admin.list_fids 0, 100
+  #
+  # Returns:
+  #
+  #   [{"fid"=>"99",
+  #     "class"=>"normal",
+  #     "domain"=>"test",
+  #     "devcount"=>"2",
+  #     "length"=>"4",
+  #     "key"=>"file_key"},
+  #    {"fid"=>"82",
+  #     "class"=>"normal",
+  #     "devcount"=>"2",
+  #     "domain"=>"test",
+  #     "length"=>"9",
+  #     "key"=>"new_new_key"}]
+
+  def list_fids(from_fid, to_fid)
+    res = @backend.list_fids :from => from_fid, :to => to_fid
+    return clean('fid_count', 'fid_', res)
+  end
+
+  ##
+  # Returns a statistics structure representing the state of mogilefs.
+  #
+  #   admin.get_stats
+  #
+  # Returns:
+  #
+  #   {"fids"=>{"max"=>"99", "count"=>"2"},
+  #    "device"=>
+  #     [{"status"=>"alive", "files"=>"2", "id"=>"1", "host"=>"rur-1"},
+  #      {"status"=>"alive", "files"=>"2", "id"=>"2", "host"=>"rur-2"}],
+  #    "replication"=>
+  #     [{"files"=>"2", "class"=>"normal", "devcount"=>"2", "domain"=>"test"}],
+  #    "file"=>[{"files"=>"2", "class"=>"normal", "domain"=>"test"}]}
+
+  def get_stats(type = 'all')
+    res = @backend.stats type => 1
+    stats = {}
+
+    stats['replication'] = clean 'replicationcount', 'replication', res, false
+    stats['file'] = clean 'filescount', 'files', res, false
+    stats['device'] = clean 'devicescount', 'devices', res, false
+
+    if res['fidmax'] or res['fidcount'] then
+      stats['fids'] = { 'max' => res['fidmax'], 'count' => res['fidcount'] }
+    end
+
+    return stats
+  end
+
+  ##
+  # Returns the domains present in the mogilefs.
+  #
+  #   admin.get_domains
+  #
+  # Returns:
+  #
+  #   {"test"=>{"normal"=>"2", "default"=>"2"}}
+
+  def get_domains
+    res = @backend.get_domains
+
+    domains = {}
+    (1..res['domains'].to_i).each do |i|
+      domain = clean "domain#{i}classes", "domain#{i}class", res, false
+      domains[res["domain#{i}"]] = Hash[*domain.map { |d| d.values}.flatten]
+    end
+
+    return domains
+  end
+
+  ##
+  # Creates a new domain named +domain+.  Returns nil if creation failed.
+
+  def create_domain(domain)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.create_domain :domain => domain
+    return res['domain'] unless res.nil?
+  end
+
+  ##
+  # Deletes +domain+.  Returns true if successful, false if not.
+
+  def delete_domain(domain)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.delete_domain :domain => domain
+    return !res.nil?
+  end
+
+  ##
+  # Creates a new class in +domain+ named +klass+ with files replicated to
+  # +mindevcount+ devices.  Returns nil on failure.
+
+  def create_class(domain, klass, mindevcount)
+    return modify_class(domain, klass, mindevcount, :create)
+  end
+
+  ##
+  # Updates class +klass+ in +domain+ to be replicated to +mindevcount+
+  # devices.  Returns nil on failure.
+
+  def update_class(domain, klass, mindevcount)
+    return modify_class(domain, klass, mindevcount, :update)
+  end
+
+  ##
+  # Removes class +klass+ from +domain+.  Returns true if successful, false if
+  # not.
+
+  def delete_class(domain, klass)
+    res = @backend.delete_class :domain => domain, :class => klass
+    return !res.nil?
+  end
+
+  ##
+  # Creates a new host named +host+.  +args+ must contain :ip and :port.
+  # Returns true if successful, false if not.
+
+  def create_host(host, args = {})
+    raise ArgumentError, "Must specify ip and port" unless \
+      args.include? :ip and args.include? :port
+
+    return modify_host(host, args, 'create')
+  end
+
+  ##
+  # Updates +host+ with +args+.  Returns true if successful, false if not.
+
+  def update_host(host, args = {})
+    return modify_host(host, args, 'update')
+  end
+
+  ##
+  # Deletes host +host+.  Returns nil on failure.
+
+  def delete_host(host)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.delete_host :host => host
+    return !res.nil?
+  end
+
+  ##
+  # Changes the device status of +device+ on +host+ to +state+ which can be
+  # 'alive', 'down', or 'dead'.
+
+  def change_device_state(host, device, state)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.set_state :host => host, :device => device, :state => state
+    return !res.nil?
+  end
+
+  protected unless defined? $TESTING
+
+  ##
+  # Modifies +klass+ on +domain+ to store files on +mindevcount+ devices via
+  # +action+.  Returns the class name if successful, nil if not.
+
+  def modify_class(domain, klass, mindevcount, action)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.send("#{action}_class", :domain => domain, :class => klass,
+                                          :mindevcount => mindevcount)
+
+    return res['class'] unless res.nil?
+  end
+
+  ##
+  # Modifies +host+ using +args+ via +action+.  Returns true if successful,
+  # false if not.
+
+  def modify_host(host, args = {}, action = 'create')
+    args[:host] = host
+    res = @backend.send "#{action}_host", args
+    return !res.nil?
+  end
+
+  ##
+  # Turns the response +res+ from the backend into an Array of Hashes from 1
+  # to res[+count+].  If +underscore+ is true then a '_' character is assumed
+  # between the prefix and the hash key value.
+  #
+  #   res = {"host1_remoteroot"=>"/mnt/mogilefs/rur-1",
+  #          "host1_hostname"=>"rur-1",
+  #          "host1_hostid"=>"1",
+  #          "host1_http_get_port"=>"",
+  #          "host1_altip"=>"",
+  #          "hosts"=>"1",
+  #          "host1_hostip"=>"",
+  #          "host1_http_port"=>"",
+  #          "host1_status"=>"alive",
+  #          "host1_altmask"=>""}
+  #   admin.clean 'hosts', 'host', res
+  # 
+  # Returns:
+  # 
+  #   [{"status"=>"alive",
+  #     "http_get_port"=>"",
+  #     "http_port"=>"",
+  #     "hostid"=>"1",
+  #     "hostip"=>"",
+  #     "hostname"=>"rur-1",
+  #     "remoteroot"=>"/mnt/mogilefs/rur-1",
+  #     "altip"=>"",
+  #     "altmask"=>""}]
+
+  def clean(count, prefix, res, underscore = true)
+    underscore = underscore ? '_' : ''
+    return (1..res[count].to_i).map do |i|
+      dev = res.select { |k,_| k =~ /^#{prefix}#{i}#{underscore}/ }.map do |k,v|
+        [k.sub(/^#{prefix}#{i}#{underscore}/, ''), v]
+      end
+      Hash[*dev.flatten]
+    end
+  end
+
+end
+

data/lib/mogilefs/backend.rb

@@ -0,0 +1,219 @@
+require 'socket'
+require 'thread'
+require 'mogilefs'
+
+##
+# MogileFS::Backend communicates with the MogileFS trackers.
+
+class MogileFS::Backend
+
+  ##
+  # Adds MogileFS commands +names+.
+
+  def self.add_command(*names)
+    names.each do |name|
+      define_method name do |*args|
+        do_request name, args.first || {}
+      end
+    end
+  end
+
+  ##
+  # The last error
+  #--
+  # TODO Use Exceptions
+
+  attr_reader :lasterr
+
+  ##
+  # The string attached to the last error
+  #--
+  # TODO Use Exceptions
+
+  attr_reader :lasterrstr
+
+  ##
+  # Creates a new MogileFS::Backend.
+  #
+  # :hosts is a required argument and must be an Array containing one or more
+  # 'hostname:port' pairs as Strings.
+  #
+  # :timeout adjusts the request timeout before an error is returned.
+
+  def initialize(args)
+    @hosts = args[:hosts]
+    raise ArgumentError, "must specify at least one host" unless @hosts
+    raise ArgumentError, "must specify at least one host" if @hosts.empty?
+    unless @hosts == @hosts.select { |h| h =~ /:\d+$/ } then
+      raise ArgumentError, ":hosts must be in 'host:port' form"
+    end
+
+    @mutex = Mutex.new
+    @timeout = args[:timeout] || 3
+    @socket = nil
+    @lasterr = nil
+    @lasterrstr = nil
+
+    @dead = {}
+  end
+
+  ##
+  # Closes this backend's socket.
+
+  def shutdown
+    @socket.close unless @socket.nil? or @socket.closed?
+    @socket = nil
+  end
+
+  # MogileFS::MogileFS commands
+
+  add_command :create_open
+  add_command :create_close
+  add_command :get_paths
+  add_command :delete
+  add_command :sleep
+  add_command :rename
+  add_command :list_keys
+
+  # MogileFS::Backend commands
+  
+  add_command :get_hosts
+  add_command :get_devices
+  add_command :list_fids
+  add_command :stats
+  add_command :get_domains
+  add_command :create_domain
+  add_command :delete_domain
+  add_command :create_class
+  add_command :update_class
+  add_command :delete_class
+  add_command :create_host
+  add_command :update_host
+  add_command :delete_host
+  add_command :set_state
+
+  private unless defined? $TESTING
+
+  ##
+  # Returns a new TCPSocket connected to +port+ on +host+.
+
+  def connect_to(host, port)
+    return TCPSocket.new(host, port)
+  end
+
+  ##
+  # Performs the +cmd+ request with +args+.
+
+  def do_request(cmd, args)
+    @mutex.synchronize do
+      request = make_request cmd, args
+
+      begin
+        bytes_sent = socket.send request, 0
+      rescue SystemCallError
+        @socket = nil
+        raise "couldn't connect to mogilefsd backend"
+      end
+
+      unless bytes_sent == request.length then
+        raise "request truncated (sent #{bytes_sent} expected #{request.length})"
+      end
+
+      readable?
+
+      return parse_response(socket.gets)
+    end
+  end
+
+  ##
+  # Makes a new request string for +cmd+ and +args+.
+
+  def make_request(cmd, args)
+    return "#{cmd} #{url_encode args}\r\n"
+  end
+
+  ##
+  # Turns the +line+ response from the server into a Hash of options, an
+  # error, or raises, as appropriate.
+
+  def parse_response(line)
+    if line =~ /^ERR\s+(\w+)\s*(.*)/ then
+      @lasterr = $1
+      @lasterrstr = $2 ? url_unescape($2) : nil
+      return nil
+    end
+
+    return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)/
+
+    raise "Invalid response from server: #{line.inspect}"
+  end
+
+  ##
+  # Raises if the socket does not become readable in +@timeout+ seconds.
+
+  def readable?
+    found = select [socket], nil, nil, @timeout
+    raise MogileFS::UnreadableSocketError if found.nil? or found.empty?
+    return true
+  end
+
+  ##
+  # Returns a socket connected to a MogileFS tracker.
+
+  def socket
+    return @socket if @socket and not @socket.closed?
+
+    now = Time.now
+
+    @hosts.sort_by { rand(3) - 1 }.each do |host|
+      next if @dead.include? host and @dead[host] > now - 5
+
+      begin
+        @socket = connect_to(*host.split(':'))
+      rescue SystemCallError
+        @dead[host] = now
+        next
+      end
+
+      return @socket
+    end
+
+    raise "couldn't connect to mogilefsd backend"
+  end
+
+  ##
+  # Turns a url params string into a Hash.
+
+  def url_decode(str)
+    pairs = str.split('&').map do |pair|
+      pair.split('=', 2).map { |v| url_unescape v }
+    end
+
+    return Hash[*pairs.flatten]
+  end
+
+  ##
+  # Turns a Hash (or Array of pairs) into a url params string.
+
+  def url_encode(params)
+    return params.map do |k,v|
+      "#{url_escape k.to_s}=#{url_escape v.to_s}"
+    end.join("&")
+  end
+
+  ##
+  # Escapes naughty URL characters.
+
+  def url_escape(str)
+    return str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
+  end
+
+  ##
+  # Unescapes naughty URL characters.
+
+  def url_unescape(str)
+    return str.gsub(/%([a-f0-9][a-f0-9])/i) { [$1.to_i(16)].pack 'C' }.tr('+', ' ')
+  end
+
+end
+