pager-mogilefs-client 1.2.1.20080519

data/lib/mogilefs/backend.rb

@@ -0,0 +1,222 @@
+require 'socket'
+require 'thread'
+require 'mogilefs'
+
+##
+# MogileFS::Backend communicates with the MogileFS trackers.
+
+class MogileFS::Backend
+
+  ##
+  # Adds MogileFS commands +names+.
+
+  def self.add_command(*names)
+    names.each do |name|
+      define_method name do |*args|
+        do_request name, args.first || {}
+      end
+    end
+  end
+
+  ##
+  # The last error
+  #--
+  # TODO Use Exceptions
+
+  attr_reader :lasterr
+
+  ##
+  # The string attached to the last error
+  #--
+  # TODO Use Exceptions
+
+  attr_reader :lasterrstr
+
+  ##
+  # Creates a new MogileFS::Backend.
+  #
+  # :hosts is a required argument and must be an Array containing one or more
+  # 'hostname:port' pairs as Strings.
+  #
+  # :timeout adjusts the request timeout before an error is returned.
+
+  def initialize(args)
+    @hosts = args[:hosts]
+    raise ArgumentError, "must specify at least one host" unless @hosts
+    raise ArgumentError, "must specify at least one host" if @hosts.empty?
+    unless @hosts == @hosts.select { |h| h =~ /:\d+$/ } then
+      raise ArgumentError, ":hosts must be in 'host:port' form"
+    end
+
+    @mutex = Mutex.new
+    @timeout = args[:timeout] || 3
+    @socket = nil
+    @lasterr = nil
+    @lasterrstr = nil
+
+    @dead = {}
+  end
+
+  ##
+  # Closes this backend's socket.
+
+  def shutdown
+    @socket.close unless @socket.nil? or @socket.closed?
+    @socket = nil
+  end
+
+  # MogileFS::MogileFS commands
+
+  add_command :create_open
+  add_command :create_close
+  add_command :get_paths
+  add_command :delete
+  add_command :sleep
+  add_command :rename
+  add_command :list_keys
+
+  # MogileFS::Backend commands
+  
+  add_command :get_hosts
+  add_command :get_devices
+  add_command :list_fids
+  add_command :stats
+  add_command :get_domains
+  add_command :create_domain
+  add_command :delete_domain
+  add_command :create_class
+  add_command :update_class
+  add_command :delete_class
+  add_command :create_host
+  add_command :update_host
+  add_command :delete_host
+  add_command :set_state
+
+  private unless defined? $TESTING
+
+  ##
+  # Returns a new TCPSocket connected to +port+ on +host+.
+
+  def connect_to(host, port)
+    return TCPSocket.new(host, port)
+  end
+
+  ##
+  # Performs the +cmd+ request with +args+.
+
+  def do_request(cmd, args)
+    @mutex.synchronize do
+      request = make_request cmd, args
+
+      begin
+        bytes_sent = socket.send request, 0
+      rescue SystemCallError
+        @socket = nil
+        raise "couldn't connect to mogilefsd backend"
+      end
+
+      unless bytes_sent == request.length then
+        raise "request truncated (sent #{bytes_sent} expected #{request.length})"
+      end
+
+      readable?
+
+      return parse_response(socket.gets)
+    end
+  end
+
+  ##
+  # Makes a new request string for +cmd+ and +args+.
+
+  def make_request(cmd, args)
+    return "#{cmd} #{url_encode args}\r\n"
+  end
+
+  ##
+  # Turns the +line+ response from the server into a Hash of options, an
+  # error, or raises, as appropriate.
+
+  def parse_response(line)
+    if line =~ /^ERR\s+(\w+)\s*(.*)/ then
+      @lasterr = $1
+      @lasterrstr = $2 ? url_unescape($2) : nil
+      return nil
+    end
+
+    return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)/
+
+    raise "Invalid response from server: #{line.inspect}"
+  end
+
+  ##
+  # Raises if the socket does not become readable in +@timeout+ seconds.
+
+  def readable?
+    found = select [socket], nil, nil, @timeout
+    if found.nil? or found.empty? then
+      peer = (@socket ? "#{@socket.peeraddr[3]}:#{@socket.peeraddr[1]} " : nil)
+      raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
+    end
+    return true
+  end
+
+  ##
+  # Returns a socket connected to a MogileFS tracker.
+
+  def socket
+    return @socket if @socket and not @socket.closed?
+
+    now = Time.now
+
+    @hosts.sort_by { rand(3) - 1 }.each do |host|
+      next if @dead.include? host and @dead[host] > now - 5
+
+      begin
+        @socket = connect_to(*host.split(':'))
+      rescue SystemCallError
+        @dead[host] = now
+        next
+      end
+
+      return @socket
+    end
+
+    raise "couldn't connect to mogilefsd backend"
+  end
+
+  ##
+  # Turns a url params string into a Hash.
+
+  def url_decode(str)
+    pairs = str.split('&').map do |pair|
+      pair.split('=', 2).map { |v| url_unescape v }
+    end
+
+    return Hash[*pairs.flatten]
+  end
+
+  ##
+  # Turns a Hash (or Array of pairs) into a url params string.
+
+  def url_encode(params)
+    return params.map do |k,v|
+      "#{url_escape k.to_s}=#{url_escape v.to_s}"
+    end.join("&")
+  end
+
+  ##
+  # Escapes naughty URL characters.
+
+  def url_escape(str)
+    return str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
+  end
+
+  ##
+  # Unescapes naughty URL characters.
+
+  def url_unescape(str)
+    return str.gsub(/%([a-f0-9][a-f0-9])/i) { [$1.to_i(16)].pack 'C' }.tr('+', ' ')
+  end
+
+end
+

data/lib/mogilefs/client.rb

@@ -0,0 +1,65 @@
+require 'mogilefs/backend'
+
+##
+# MogileFS::Client is the MogileFS client base class.  Concrete clients like
+# MogileFS::MogileFS and MogileFS::Admin are implemented atop this one to do
+# real work.
+
+class MogileFS::Client
+
+  ##
+  # The backend connection for this client
+
+  attr_reader :backend
+
+  attr_accessor :hosts if defined? $TESTING
+
+  ##
+  # Creates a new Client.  See MogileFS::Backend#initialize for how to specify
+  # hosts.  If :readonly is set to true, the client will not modify anything
+  # on the server.
+  #
+  #   MogileFS::Client.new :hosts => ['kaa:6001', 'ziz:6001'], :readonly => true
+
+  def initialize(args)
+    @hosts = args[:hosts]
+    @readonly = args[:readonly] ? true : false
+    @timeout = args[:timeout]
+
+    reload
+  end
+
+  ##
+  # Creates a new MogileFS::Backend.
+
+  def reload
+    @backend = MogileFS::Backend.new :hosts => @hosts, :timeout => @timeout
+  end
+
+  ##
+  # The last error reported by the backend.
+  #--
+  # TODO use Exceptions
+
+  def err
+    @backend.lasterr
+  end
+
+  ##
+  # The last error message reported by the backend.
+  #--
+  # TODO use Exceptions
+
+  def errstr
+    @backend.lasterrstr
+  end
+
+  ##
+  # Is this a read-only client?
+
+  def readonly?
+    return @readonly
+  end
+
+end
+

data/lib/mogilefs/httpfile.rb

@@ -0,0 +1,118 @@
+require 'fcntl'
+require 'socket'
+require 'stringio'
+require 'uri'
+require 'mogilefs/backend'
+
+##
+# HTTPFile wraps up the new file operations for storing files onto an HTTP
+# storage node.
+#
+# You really don't want to create an HTTPFile by hand.  Instead you want to
+# create a new file using MogileFS::MogileFS.new_file.
+#
+# WARNING! HTTP mode is completely untested as I cannot make it work on
+# FreeBSD.  Please send patches/tests if you find bugs.
+#--
+# TODO dup'd content in MogileFS::NFSFile
+
+class MogileFS::HTTPFile < StringIO
+
+  ##
+  # The path this file will be stored to.
+
+  attr_reader :path
+
+  ##
+  # The key for this file.  This key won't represent a real file until you've
+  # called #close.
+
+  attr_reader :key
+
+  ##
+  # The class of this file.
+
+  attr_reader :class
+
+  ##
+  # Works like File.open.  Use MogileFS::MogileFS#new_file instead of this
+  # method.
+
+  def self.open(*args)
+    fp = new(*args)
+
+    return fp unless block_given?
+
+    begin
+      yield fp
+    ensure
+      fp.close
+    end
+  end
+
+  ##
+  # Creates a new HTTPFile with MogileFS-specific data.  Use
+  # MogileFS::MogileFS#new_file instead of this method.
+
+  def initialize(mg, fid, path, devid, klass, key, dests, content_length)
+    super ''
+    @mg = mg
+    @fid = fid
+    @path = path
+    @devid = devid
+    @klass = klass
+    @key = key
+
+    @dests = dests.map { |(_,u)| URI.parse u }
+    @tried = {}
+
+    @socket = nil
+  end
+
+  ##
+  # Closes the file handle and marks it as closed in MogileFS.
+
+  def close
+    connect_client
+    
+    resp = @client.put(@path.request_uri, :body => string)    
+    raise "HTTP response status from upload: #{resp.http_status}" unless resp.http_status.to_i == 200
+      
+    @mg.backend.create_close(:fid => @fid, :devid => @devid,
+                             :domain => @mg.domain, :key => @key,
+                             :path => @path, :size => length)
+    return nil
+  end
+
+  private
+
+  def connected?
+    return !(@client.nil?)
+  end
+
+  def connect_client
+    return @client if connected?
+
+    next_path
+
+    if @path.nil? then
+      @tried.clear
+      next_path
+      raise 'Unable to open socket to storage node' if @path.nil?
+    end
+
+    @client = RFuzz::HttpClient.new(@path.host, @path.port)
+  end
+
+  def next_path
+    @path = nil
+    @dests.each do |dest|
+      unless @tried.include? dest then
+        @path = dest
+        return
+      end
+    end
+  end
+
+end
+

data/lib/mogilefs/mogilefs.rb

@@ -0,0 +1,237 @@
+require 'open-uri'
+require 'timeout'
+
+require 'rfuzz/client'
+
+require 'mogilefs/client'
+require 'mogilefs/nfsfile'
+
+##
+# Timeout error class.
+
+class MogileFS::Timeout < Timeout::Error; end
+
+##
+# MogileFS File manipulation client.
+
+class MogileFS::MogileFS < MogileFS::Client
+
+  ##
+  # The path to the local MogileFS mount point if you are using NFS mode.
+
+  attr_reader :root
+
+  ##
+  # The domain of keys for this MogileFS client.
+
+  attr_reader :domain
+
+  ##
+  # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
+  # :domain specifying the domain of this client.  A key :root will be used to
+  # specify the root of the NFS file system.
+
+  def initialize(args = {})
+    @domain = args[:domain]
+    @root = args[:root]
+
+    raise ArgumentError, "you must specify a domain" unless @domain
+
+    super
+  end
+
+  ##
+  # Enumerates keys starting with +key+.
+
+  def each_key(prefix)
+    after = nil
+
+    keys, after = list_keys prefix
+
+    until keys.empty? do
+      keys.each { |k| yield k }
+      keys, after = list_keys prefix, after
+    end
+
+    return nil
+  end
+
+  ##
+  # Retrieves the contents of +key+.
+
+  def get_file_data(key)
+    paths = get_paths key
+
+    return nil unless paths
+
+    paths.each do |path|
+      next unless path
+      case path
+      when /^http:\/\// then
+        begin
+          path = URI.parse path
+          data = timeout(5, MogileFS::Timeout) { 
+            RFuzz::HttpClient.new(path.host, path.port).get(path).http_body
+          }
+          return data
+        rescue MogileFS::Timeout, RFuzz::HttpClientError, RFuzz::HttpClientParserError
+          next
+        end
+      else
+        next unless File.exist? path
+        return File.read(path)
+      end
+    end
+
+    return nil
+  end
+
+  ##
+  # Get the paths for +key+.
+
+  def get_paths(key, noverify = true, zone = nil)
+    noverify = noverify ? 1 : 0
+    res = @backend.get_paths(:domain => @domain, :key => key,
+                             :noverify => noverify, :zone => zone)
+    
+    return nil if res.nil? and @backend.lasterr == 'unknown_key'
+    paths = (1..res['paths'].to_i).map { |i| res["path#{i}"] }
+    return paths if paths.empty?
+    return paths if paths.first =~ /^http:\/\//
+    return paths.map { |path| File.join @root, path }
+  end
+
+  ##
+  # Creates a new file +key+ in +klass+.  +bytes+ is currently unused.
+  #
+  # The +block+ operates like File.open.
+
+  def new_file(key, klass, bytes = 0, &block) # :yields: file
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.create_open(:domain => @domain, :class => klass,
+                               :key => key, :multi_dest => 1)
+
+    raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? # HACK
+
+    dests = nil
+
+    if res.include? 'dev_count' then # HACK HUH?
+      dests = (1..res['dev_count'].to_i).map do |i|
+        [res["devid_#{i}"], res["path_#{i}"]]
+      end
+    else
+      # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
+      # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
+      # 0x0060:  3932 2e31 3638 2e31 2e37 323a 3735 3030  92.168.1.72:7500
+      # 0x0070:  2f64 6576 312f 302f 3030 302f 3030 302f  /dev1/0/000/000/
+      # 0x0080:  3030 3030 3030 3030 3033 2e66 6964 0d0a  0000000003.fid..
+
+      dests = [[res['devid'], res['path']]]
+    end
+
+    dest = dests.first
+    devid, path = dest
+
+    case path
+    when /^http:\/\// then
+      MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
+                              dests, bytes, &block)
+    else
+      MogileFS::NFSFile.open(self, res['fid'], path, devid, klass, key, &block)
+    end
+  end
+
+  ##
+  # Copies the contents of +file+ into +key+ in class +klass+.  +file+ can be
+  # either a file name or an object that responds to #read.
+
+  def store_file(key, klass, file)
+    raise 'readonly mogilefs' if readonly?
+
+    new_file key, klass do |mfp|
+      if file.respond_to? :read then
+        return copy(file, mfp)
+      else
+        return File.open(file) { |fp| copy(fp, mfp) }
+      end
+    end
+  end
+
+  ##
+  # Stores +content+ into +key+ in class +klass+.
+
+  def store_content(key, klass, content)
+    raise 'readonly mogilefs' if readonly?
+
+    new_file key, klass do |mfp|
+      mfp << content
+    end
+
+    return content.length
+  end
+
+  ##
+  # Removes +key+.
+
+  def delete(key)
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.delete :domain => @domain, :key => key
+
+    if res.nil? and @backend.lasterr != 'unknown_key' then
+      raise "unable to delete #{key}: #{@backend.lasterr}"
+    end
+  end
+
+  ##
+  # Sleeps +duration+.
+
+  def sleep(duration)
+    @backend.sleep :duration => duration
+  end
+
+  ##
+  # Renames a key +from+ to key +to+.
+
+  def rename(from, to)
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.rename :domain => @domain, :from_key => from, :to_key => to
+
+    if res.nil? and @backend.lasterr != 'unknown_key' then
+      raise "unable to rename #{from_key} to #{to_key}: #{@backend.lasterr}"
+    end
+  end
+
+  ##
+  # Lists keys starting with +prefix+ follwing +after+ up to +limit+.  If
+  # +after+ is nil the list starts at the beginning.
+
+  def list_keys(prefix, after = nil, limit = 1000)
+    res = @backend.list_keys(:domain => domain, :prefix => prefix,
+                             :after => after, :limit => limit)
+
+    return nil if res.nil?
+
+    keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
+
+    return keys, res['next_after']
+  end
+
+  private
+
+  def copy(from, to) # HACK use FileUtils
+    bytes = 0
+
+    until from.eof? do
+      chunk = from.read 8192
+      to.write chunk
+      bytes += chunk.length
+    end
+
+    return bytes
+  end
+
+end
+