mogilefs-client 1.0.1

data/lib/mogilefs/client.rb

@@ -0,0 +1,64 @@
+require 'mogilefs/backend'
+
+##
+# MogileFS::Client is the MogileFS client base class.  Concrete clients like
+# MogileFS::MogileFS and MogileFS::Admin are implemented atop this one to do
+# real work.
+
+class MogileFS::Client
+
+  ##
+  # The backend connection for this client
+
+  attr_reader :backend
+
+  attr_accessor :hosts if defined? $TESTING
+
+  ##
+  # Creates a new Client.  See MogileFS::Backend#initialize for how to specify
+  # hosts.  If :readonly is set to true, the client will not modify anything
+  # on the server.
+  #
+  #   MogileFS::Client.new :hosts => ['kaa:6001', 'ziz:6001'], :readonly => true
+
+  def initialize(args)
+    @hosts = args[:hosts]
+    @readonly = args[:readonly] ? true : false
+
+    reload
+  end
+
+  ##
+  # Creates a new MogileFS::Backend.
+
+  def reload
+    @backend = MogileFS::Backend.new :hosts => @hosts
+  end
+
+  ##
+  # The last error reported by the backend.
+  #--
+  # TODO use Exceptions
+
+  def err
+    @backend.lasterr
+  end
+
+  ##
+  # The last error message reported by the backend.
+  #--
+  # TODO use Exceptions
+
+  def errstr
+    @backend.lasterrstr
+  end
+
+  ##
+  # Is this a read-only client?
+
+  def readonly?
+    return @readonly
+  end
+
+end
+

data/lib/mogilefs/httpfile.rb

@@ -0,0 +1,144 @@
+require 'fcntl'
+require 'socket'
+require 'stringio'
+require 'uri'
+require 'mogilefs/backend'
+
+##
+# HTTPFile wraps up the new file operations for storing files onto an HTTP
+# storage node.
+#
+# You really don't want to create an HTTPFile by hand.  Instead you want to
+# create a new file using MogileFS::MogileFS.new_file.
+#
+# WARNING! HTTP mode is completely untested as I cannot make it work on
+# FreeBSD.  Please send patches/tests if you find bugs.
+#--
+# TODO dup'd content in MogileFS::NFSFile
+
+class MogileFS::HTTPFile < StringIO
+
+  ##
+  # The path this file will be stored to.
+
+  attr_reader :path
+
+  ##
+  # The key for this file.  This key won't represent a real file until you've
+  # called #close.
+
+  attr_reader :key
+
+  ##
+  # The class of this file.
+
+  attr_reader :class
+
+  ##
+  # Works like File.open.  Use MogileFS::MogileFS#new_file instead of this
+  # method.
+
+  def self.open(*args)
+    fp = new(*args)
+
+    return fp unless block_given?
+
+    begin
+      yield fp
+    ensure
+      fp.close
+    end
+  end
+
+  ##
+  # Creates a new HTTPFile with MogileFS-specific data.  Use
+  # MogileFS::MogileFS#new_file instead of this method.
+
+  def initialize(mg, fid, path, devid, klass, key, dests, content_length)
+    super ''
+    @mg = mg
+    @fid = fid
+    @path = path
+    @devid = devid
+    @klass = klass
+    @key = key
+
+    @dests = dests.map { |(_,u)| URI.parse u }
+    @tried = {}
+
+    @socket = nil
+  end
+
+  ##
+  # Closes the file handle and marks it as closed in MogileFS.
+
+  def close
+    connect_socket
+
+    @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-length: #{length}\r\n\r\n#{string}"
+
+    if connected? then
+      line = @socket.gets
+      raise 'Unable to read response line from server' if line.nil?
+
+      if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
+        status = Integer $1
+        case status
+        when 200..299 then # success!
+        else
+          found_header = false
+          body = []
+          line = @socket.gets
+          until line.nil? do
+            line.strip
+            found_header = true if line.nil?
+            next unless found_header
+            body << " #{line}"
+          end
+          body = body[0, 512] if body.length > 512
+          raise "HTTP response status from upload: #{body}"
+        end
+      else
+        raise "Response line not understood: #{line}"
+      end
+      @socket.close
+    end
+
+    @mg.backend.create_close(:fid => @fid, :devid => @devid,
+                             :domain => @mg.domain, :key => @key,
+                             :path => @path, :size => length)
+    return nil
+  end
+
+  private
+
+  def connected?
+    return !(@socket.nil? or @socket.closed?)
+  end
+
+  def connect_socket
+    return @socket if connected?
+
+    next_path
+
+    if @path.nil? then
+      @tried.clear
+      next_path
+      raise 'Unable to open socket to storage node' if @path.nil?
+    end
+
+    @socket = TCPSocket.new @path.host, @path.port
+  end
+
+  def next_path
+    @path = nil
+    @dests.each do |dest|
+      unless @tried.include? dest then
+        @path = dest
+        return
+      end
+    end
+  end
+
+end
+

data/lib/mogilefs/mogilefs.rb

@@ -0,0 +1,214 @@
+require 'open-uri'
+require 'timeout'
+
+require 'mogilefs/client'
+require 'mogilefs/nfsfile'
+
+##
+# Timeout error class.
+
+class MogileFS::Timeout < Timeout::Error; end
+
+##
+# MogileFS File manipulation client.
+
+class MogileFS::MogileFS < MogileFS::Client
+
+  ##
+  # The path to the local MogileFS mount point if you are using NFS mode.
+
+  attr_reader :root
+
+  ##
+  # The domain of keys for this MogileFS client.
+
+  attr_reader :domain
+
+  ##
+  # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
+  # :domain specifying the domain of this client.  A key :root will be used to
+  # specify the root of the NFS file system.
+
+  def initialize(args = {})
+    @domain = args[:domain]
+    @root = args[:root]
+
+    raise ArgumentError, "you must specify a domain" unless @domain
+
+    super
+  end
+
+  ##
+  # Retrieves the contents of +key+.
+
+  def get_file_data(key)
+    get_paths(key).each do |path|
+      next unless path
+      case path
+      when /^http:\/\// then
+        begin
+          puts "path: #{path}"
+          path = URI.parse path
+          data = timeout(5, MogileFS::Timeout) { path.read }
+          puts "data: #{data}"
+        rescue MogileFS::Timeout
+          next
+        end
+      else
+        next unless File.exist? path
+        return File.read(path)
+      end
+    end
+
+    return nil
+  end
+
+  ##
+  # Get the paths for +key+.
+
+  def get_paths(key, noverify = true, zone = nil)
+    noverify = noverify ? 1 : 0
+    res = @backend.get_paths(:domain => @domain, :key => key,
+                             :noverify => noverify, :zone => zone)
+    
+    return nil if res.nil? and @backend.lasterr == 'unknown_key'
+    paths = (1..res['paths'].to_i).map { |i| res["path#{i}"] }
+    return paths if paths.empty?
+    return paths if paths.first =~ /^http:\/\//
+    return paths.map { |path| File.join @root, path }
+  end
+
+  ##
+  # Creates a new file +key+ in +klass+.  +bytes+ is currently unused.
+  #
+  # The +block+ operates like File.open.
+
+  def new_file(key, klass, bytes = 0, &block) # :yields: file
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.create_open(:domain => @domain, :class => klass,
+                               :key => key, :multi_dest => 1)
+
+    raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? # HACK
+
+    dests = nil
+
+    if res.include? 'dev_count' then # HACK HUH?
+      dests = (1..res['dev_count'].to_i).map do |i|
+        [res["devid_#{i}"], res["path_#{i}"]]
+      end
+    else
+      # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
+      # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
+      # 0x0060:  3932 2e31 3638 2e31 2e37 323a 3735 3030  92.168.1.72:7500
+      # 0x0070:  2f64 6576 312f 302f 3030 302f 3030 302f  /dev1/0/000/000/
+      # 0x0080:  3030 3030 3030 3030 3033 2e66 6964 0d0a  0000000003.fid..
+
+      dests = [[res['devid'], res['path']]]
+    end
+
+    dest = dests.first
+    devid, path = dest
+
+    case path
+    when /^http:\/\// then
+      MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
+                              dests, bytes, &block)
+    else
+      MogileFS::NFSFile.open(self, res['fid'], path, devid, klass, key, &block)
+    end
+  end
+
+  ##
+  # Copies the contents of +file+ into +key+ in class +klass+.  +file+ can be
+  # either a file name or an object that responds to #read.
+
+  def store_file(key, klass, file)
+    raise 'readonly mogilefs' if readonly?
+
+    new_file key, klass do |mfp|
+      if file.respond_to? :read then
+        return copy(file, mfp)
+      else
+        return File.open(file) { |fp| copy(fp, mfp) }
+      end
+    end
+  end
+
+  ##
+  # Stores +content+ into +key+ in class +klass+.
+
+  def store_content(key, klass, content)
+    raise 'readonly mogilefs' if readonly?
+
+    new_file key, klass do |mfp|
+      mfp << content
+    end
+
+    return content.length
+  end
+
+  ##
+  # Removes +key+.
+
+  def delete(key)
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.delete :domain => @domain, :key => key
+
+    if res.nil? and @backend.lasterr != 'unknown_key' then
+      raise "unable to delete #{key}: #{@backend.lasterr}"
+    end
+  end
+
+  ##
+  # Sleeps +duration+.
+
+  def sleep(duration)
+    @backend.sleep :duration => duration
+  end
+
+  ##
+  # Renames a key +from+ to key +to+.
+
+  def rename(from, to)
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.rename :domain => @domain, :from_key => from, :to_key => to
+
+    if res.nil? and @backend.lasterr != 'unknown_key' then
+      raise "unable to rename #{from_key} to #{to_key}: #{@backend.lasterr}"
+    end
+  end
+
+  ##
+  # Lists keys starting with +prefix+ follwing +after+ up to +limit+.  If
+  # +after+ is nil the list starts at the beginning.
+
+  def list_keys(prefix, after = nil, limit = 1000)
+    res = @backend.list_keys(:domain => domain, :prefix => prefix,
+                             :after => after, :limit => limit)
+
+    return nil if res.nil?
+
+    keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
+
+    return keys, res['next_after']
+  end
+
+  private
+
+  def copy(from, to) # HACK use FileUtils
+    bytes = 0
+
+    until from.eof? do
+      chunk = from.read 8192
+      to.write chunk
+      bytes += chunk.length
+    end
+
+    return bytes
+  end
+
+end
+

data/lib/mogilefs/nfsfile.rb

@@ -0,0 +1,81 @@
+require 'mogilefs/backend'
+
+##
+# NFSFile wraps up the new file operations for storing files onto an NFS
+# storage node.
+#
+# You really don't want to create an NFSFile by hand.  Instead you want to
+# create a new file using MogileFS::MogileFS.new_file.
+
+class MogileFS::NFSFile < File
+
+  ##
+  # The path of this file not including the local mount point.
+
+  attr_reader :path
+
+  ##
+  # The key for this file.  This key won't represent a real file until you've
+  # called #close.
+
+  attr_reader :key
+
+  ##
+  # The class of this file.
+
+  attr_reader :class
+
+  class << self
+
+    ##
+    # Wraps up File.new with MogileFS-specific data.  Use
+    # MogileFS::MogileFS#new_file instead of this method.
+
+    def new(mg, fid, path, devid, klass, key)
+      fp = super join(mg.root, path), 'w+'
+      fp.send :setup, mg, fid, path, devid, klass, key
+      return fp
+    end
+
+    ##
+    # Wraps up File.open with MogileFS-specific data.  Use
+    # MogileFS::MogileFS#new_file instead of this method.
+
+    def open(mg, fid, path, devid, klass, key, &block)
+      fp = new mg, fid, path, devid, klass, key
+
+      return fp if block.nil?
+
+      begin
+        yield fp
+      ensure
+        fp.close
+      end
+    end
+
+  end
+
+  ##
+  # Closes the file handle and marks it as closed in MogileFS.
+
+  def close
+    super
+    @mg.backend.create_close(:fid => @fid, :devid => @devid,
+                             :domain => @mg.domain, :key => @key,
+                             :path => @path)
+    return nil
+  end
+
+  private
+
+  def setup(mg, fid, path, devid, klass, key)
+    @mg = mg
+    @fid = fid
+    @path = path
+    @devid = devid
+    @klass = klass
+    @key = key
+  end
+
+end
+