mogilefs-client 1.2.1 → 1.3.0

data/History.txt

@@ -1,3 +1,21 @@
+= 1.3.0
+
+* Fixed MogileFS#rename.  Bug #14465 submitted by Justin Dossey.
+* Removed infinite loop in MogileFS::HTTPFile#store_file.  Patch #13789
+  submitted by  Andy Lo-A-Foe.
+* Made MogileFS#get_file_data timeout configurable.  Bug #13490 submitted by
+  Andy Lo-A-Foe.
+* Add MogileFS#size.  Feature Request #14484 submitted by Justin Dossey.
+* Fix MogileFS#get_file_data to return the data for HTTP mode.  Bug #7133
+  submitted by John Wanko.
+* New maintainer: Eric Wong
+* Add `mog' command-line tool as a demo/example
+* Lower memory consumption with large files
+* Allow get_file_data to accept a block for large files
+* Fix each_keys loop termination condition
+* Apply error handling patch from Matthew Willson.  Bug #15987
+* Merge large file patch from Andy Lo-A-Foe.  Bug #13764
+
 = 1.2.1
 
 * Switched to Hoe.

data/Manifest.txt

@@ -3,6 +3,7 @@ LICENSE.txt
 Manifest.txt
 README.txt
 Rakefile
+bin/mog
 lib/mogilefs.rb
 lib/mogilefs/admin.rb
 lib/mogilefs/backend.rb
@@ -11,6 +12,7 @@ lib/mogilefs/httpfile.rb
 lib/mogilefs/mogilefs.rb
 lib/mogilefs/nfsfile.rb
 lib/mogilefs/pool.rb
+lib/mogilefs/util.rb
 test/setup.rb
 test/test_admin.rb
 test/test_backend.rb

data/Rakefile

@@ -6,12 +6,15 @@ require 'mogilefs'
 
 Hoe.new 'mogilefs-client', MogileFS::VERSION do |p|
   p.rubyforge_name = 'seattlerb'
-  p.author = 'Eric Hodel'
-  p.email = 'drbrain@segment7.net'
+  p.author = [ 'Eric Wong', 'Eric Hodel' ]
+  p.email = 'normalperson@yhbt.net' # (Eric Wong)
+  # p.email = 'drbrain@segment7.net' # (Eric Hodel)
   p.summary = p.paragraphs_of('README.txt', 1).first
   p.description = p.paragraphs_of('README.txt', 9).first
   p.url = p.paragraphs_of('README.txt', 5).first
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+
+  p.extra_dev_deps << ['ZenTest', '>= 3.6.1']
 end
 
 # vim: syntax=Ruby

data/bin/mog

@@ -0,0 +1,201 @@
+#!/usr/bin/env ruby
+require 'mogilefs'
+require 'optparse'
+
+trap('INT') { exit 130 }
+trap('PIPE') { exit 0 }
+
+# this is to be compatible with config files used by the Perl tools
+def parse_config_file!(path, overwrite = false)
+  dest = {}
+  File.open(path).each_line do |line|
+    line.strip!
+    if /^(domain|class)\s*=\s*(\S+)/.match(line)
+      dest[$1.to_sym] = $2
+    elsif m = /^(?:trackers|hosts)\s*=\s*(.*)/.match(line)
+      dest[:hosts] = $1.split(/\s*,\s*/)
+    else
+      STDERR.puts "Ignored configuration line: #{line}" unless /^#/.match(line)
+    end
+  end
+  dest
+end
+
+# parse the default config file if one exists
+def_file = File.expand_path("~/.mogilefs-client.conf")
+def_cfg = File.exist?(def_file) ? parse_config_file!(def_file) : {}
+
+# parse the command-line first, these options take precedence over all else
+cli_cfg = {}
+config_file = nil
+ls_l = false
+ls_h = false
+test = {}
+
+ARGV.options do |x|
+  x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
+  x.separator ''
+
+  x.on('-c', '--config=/path/to/config',
+       'config file to load') { |file| config_file = file }
+
+  x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
+       'hostnames/IP addresses of trackers') do |trackers|
+    cli_cfg[:hosts] = trackers
+  end
+
+  x.on('-e', 'True if key exists') { test[:e] = true }
+
+  x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
+  x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
+  x.on('-l', "long listing format (`ls' command)") { ls_l = true }
+  x.on('-h', '--human-readable',
+       "print sizes in human-readable format (`ls' command)") { ls_h = true }
+
+  x.separator ''
+  x.on('--help', 'Show this help message.') { puts x; exit }
+  x.parse!
+end
+
+# parse the config file specified at the command-line
+file_cfg = config_file ? parse_config_file!(config_file, true) : {}
+
+# read environment variables, too.  This Ruby API favors the term
+# "hosts", however upstream MogileFS teminology favors "trackers" instead.
+# Favor the term more consistent with what the MogileFS inventors used.
+env_cfg = {}
+if ENV["MOG_TRACKERS"]
+  env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
+end
+if ENV["MOG_HOSTS"] && env_cfg[:hosts].empty?
+  env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
+end
+env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
+env_cfg[:class] = ENV["MOG_CLASS"] if ENV["MOG_CLASS"]
+
+# merge the configs, favoring them in order specified:
+cfg = {}.merge(def_cfg).merge(env_cfg).merge(file_cfg).merge(cli_cfg)
+
+# error-checking
+err = []
+err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
+err << "domain must be specified" unless cfg[:domain]
+if err.any?
+  STDERR.puts "Errors:\n  #{err.join("\n  ")}"
+  STDERR.puts ARGV.options
+  exit 1
+end
+
+unless cmd = ARGV.shift
+  STDERR.puts ARGV.options
+  exit 1
+end
+
+include MogileFS::Util
+mg = MogileFS::MogileFS.new(cfg)
+
+begin
+  case cmd
+  when 'cp'
+    filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
+    key = ARGV.shift or raise ArgumentError, '<filename> <key>'
+    ARGV.shift and raise ArgumentError, '<filename> <key>'
+    cfg[:class] or raise ArgumentError, 'E: --class must be specified'
+    mg.store_file(key, cfg[:class], filename)
+  when 'cat'
+    ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
+    ARGV.each { |key| mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) } }
+  when 'ls'
+    prefixes = ARGV.empty? ? [ nil ] : ARGV
+    prefixes.each do |prefix|
+      mg.each_key(prefix) do |key|
+        if ls_l
+          path_nr = "% 2d" % mg.get_paths(key).size
+          size = mg.size(key)
+          if ls_h && size > 1024
+            suff = ''
+            %w(K M G).each do |s|
+              size /= 1024.0
+              suff = s
+              break if size <= 1024
+            end
+            size = sprintf("%.1f%s", size, suff)
+          else
+            size = size.to_s
+          end
+          size = (' ' * (12 - size.length)) << size # right justify
+          puts [ path_nr, size, key ].pack("A4 A16 A32")
+        else
+          puts key
+        end
+      end
+    end
+  when 'rm'
+    ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
+    ARGV.each { |key| mg.delete(key) }
+  when 'mv'
+    from = ARGV.shift or raise ArgumentError, '<from> <to>'
+    to = ARGV.shift or raise ArgumentError, '<from> <to>'
+    ARGV.shift and raise ArgumentError, '<from> <to>'
+    mg.rename(from, to)
+  when 'stat' # this outputs a RFC822-like format
+    ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
+    ARGV.each_with_index do |key, i|
+      if size = mg.size(key)
+        puts "Key: #{key}"
+        puts "Size: #{size}"
+        mg.get_paths(key).each_with_index do |path,i|
+          puts "URL-#{i}: #{path}"
+        end
+        puts ""
+      else
+        STDERR.puts "No such key: #{key}"
+      end
+    end
+  when 'tee'
+    require 'tempfile'
+    key = ARGV.shift or raise ArgumentError, '<key>'
+    ARGV.shift and raise ArgumentError, '<key>'
+    cfg[:class] or raise ArgumentError, 'E: --class must be specified'
+    buf = ''
+    tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
+    at_exit { tmp.unlink }
+    begin
+      sysrwloop(STDIN, tmp)
+      mg.store_file(key, cfg[:class], tmp.path)
+    ensure
+      tmp.close
+    end
+  when 'test'
+    truth, ok = true, nil
+    raise ArgumentError, "-e must be specified" unless (test.size == 1)
+
+    truth, key = case ARGV.size
+    when 1
+      [ true, ARGV[0] ]
+    when 2
+      if ARGV[0] != "!"
+        raise ArgumentError, "#{ARGV[0]}: binary operator expected"
+      end
+      [ false, ARGV[1] ]
+    else
+      raise ArgumentError, "Too many arguments"
+    end
+
+    paths = mg.get_paths(key)
+    if test[:e]
+      ok = !!(paths && paths.size > 0)
+    else
+      raise ArgumentError, "Unknown flag: -#{test.keys.first}"
+    end
+
+    truth or ok = ! ok
+    exit ok ? 0 : 1
+  else
+    raise ArgumentError, "Unknown command: #{cmd}"
+  end
+rescue ArgumentError => err
+  STDERR.puts "Usage: #{$0} #{cmd} #{err.message}"
+  exit 1
+end
+exit 0

data/lib/mogilefs.rb

@@ -6,7 +6,7 @@
 
 module MogileFS
 
-  VERSION = '1.2.1'
+  VERSION = '1.3.0'
 
   ##
   # Raised when a socket remains unreadable for too long.

data/lib/mogilefs/backend.rb

@@ -155,6 +155,7 @@ class MogileFS::Backend
     found = select [socket], nil, nil, @timeout
     if found.nil? or found.empty? then
       peer = (@socket ? "#{@socket.peeraddr[3]}:#{@socket.peeraddr[1]} " : nil)
+      socket.close  # we DO NOT want the response we timed out waiting for, to crop up later on, on the same socket, intersperesed with a subsequent request! so, we close the socket if it times out like this
       raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
     end
     return true

data/lib/mogilefs/httpfile.rb

@@ -3,6 +3,7 @@ require 'socket'
 require 'stringio'
 require 'uri'
 require 'mogilefs/backend'
+require 'mogilefs/util'
 
 ##
 # HTTPFile wraps up the new file operations for storing files onto an HTTP
@@ -17,6 +18,7 @@ require 'mogilefs/backend'
 # TODO dup'd content in MogileFS::NFSFile
 
 class MogileFS::HTTPFile < StringIO
+  include MogileFS::Util
 
   ##
   # The path this file will be stored to.
@@ -34,6 +36,11 @@ class MogileFS::HTTPFile < StringIO
 
   attr_reader :class
 
+  ##
+  # The bigfile name in case we have file > 256M
+
+  attr_accessor :bigfile
+
   ##
   # Works like File.open.  Use MogileFS::MogileFS#new_file instead of this
   # method.
@@ -62,6 +69,7 @@ class MogileFS::HTTPFile < StringIO
     @devid = devid
     @klass = klass
     @key = key
+    @bigfile = nil
 
     @dests = dests.map { |(_,u)| URI.parse u }
     @tried = {}
@@ -75,7 +83,17 @@ class MogileFS::HTTPFile < StringIO
   def close
     connect_socket
 
-    @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-length: #{length}\r\n\r\n#{string}"
+    file_size = nil
+    if @bigfile
+      # Don't try to run out of memory
+      fp = File.open(@bigfile)
+      file_size = File.size(@bigfile)
+      @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-Length: #{file_size}\r\n\r\n"
+      sysrwloop(fp, @socket)
+      fp.close
+    else
+      @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n#{string}"
+    end
 
     if connected? then
       line = @socket.gets
@@ -86,27 +104,19 @@ class MogileFS::HTTPFile < StringIO
         case status
         when 200..299 then # success!
         else
-          found_header = false
-          body = []
-          line = @socket.gets
-          until line.nil? do
-            line.strip
-            found_header = true if line.nil?
-            next unless found_header
-            body << " #{line}"
-          end
-          body = body[0, 512] if body.length > 512
-          raise "HTTP response status from upload: #{body}"
+          raise "HTTP response status from upload: #{status}"
         end
       else
         raise "Response line not understood: #{line}"
       end
+
       @socket.close
     end
 
     @mg.backend.create_close(:fid => @fid, :devid => @devid,
                              :domain => @mg.domain, :key => @key,
                              :path => @path, :size => length)
+    return file_size if @bigfile
     return nil
   end
 

data/lib/mogilefs/mogilefs.rb

@@ -1,4 +1,5 @@
 require 'open-uri'
+require 'net/http'
 require 'timeout'
 
 require 'mogilefs/client'
@@ -24,6 +25,11 @@ class MogileFS::MogileFS < MogileFS::Client
 
   attr_reader :domain
 
+  ##
+  # The timeout for get_file_data.  Defaults to five seconds.
+
+  attr_accessor :get_file_data_timeout
+
   ##
   # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
   # :domain specifying the domain of this client.  A key :root will be used to
@@ -33,6 +39,8 @@ class MogileFS::MogileFS < MogileFS::Client
     @domain = args[:domain]
     @root = args[:root]
 
+    @get_file_data_timeout = 5
+
     raise ArgumentError, "you must specify a domain" unless @domain
 
     super
@@ -46,7 +54,7 @@ class MogileFS::MogileFS < MogileFS::Client
 
     keys, after = list_keys prefix
 
-    until keys.empty? do
+    until keys.nil? or keys.empty? do
       keys.each { |k| yield k }
       keys, after = list_keys prefix, after
     end
@@ -57,7 +65,7 @@ class MogileFS::MogileFS < MogileFS::Client
   ##
   # Retrieves the contents of +key+.
 
-  def get_file_data(key)
+  def get_file_data(key, &block)
     paths = get_paths key
 
     return nil unless paths
@@ -68,8 +76,21 @@ class MogileFS::MogileFS < MogileFS::Client
       when /^http:\/\// then
         begin
           path = URI.parse path
-          data = timeout(5, MogileFS::Timeout) { path.read }
-          return data
+
+          if block_given?
+            sock = nil
+            timeout @get_file_data_timeout, MogileFS::Timeout do
+              sock = TCPSocket.new(path.host, path.port)
+              sock.sync = true
+              sock.syswrite("GET #{path.request_uri} HTTP/1.0\r\n\r\n")
+              buf = sock.recv(4096, Socket::MSG_PEEK)
+              head, body = buf.split(/\r\n\r\n/, 2)
+              head = sock.recv(head.size + 4)
+            end
+            return yield(sock)
+          else
+            return path.read
+          end
         rescue MogileFS::Timeout
           next
         end
@@ -108,7 +129,7 @@ class MogileFS::MogileFS < MogileFS::Client
     res = @backend.create_open(:domain => @domain, :class => klass,
                                :key => key, :multi_dest => 1)
 
-    raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? # HACK
+    raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? || res == {} # HACK
 
     dests = nil
 
@@ -130,6 +151,8 @@ class MogileFS::MogileFS < MogileFS::Client
     devid, path = dest
 
     case path
+    when nil, '' then
+      raise 'Empty path for mogile upload'
     when /^http:\/\// then
       MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
                               dests, bytes, &block)
@@ -146,10 +169,15 @@ class MogileFS::MogileFS < MogileFS::Client
     raise 'readonly mogilefs' if readonly?
 
     new_file key, klass do |mfp|
-      if file.respond_to? :read then
-        return copy(file, mfp)
+      if file.respond_to? :sysread then
+        return sysrwloop(file, mfp)
       else
-        return File.open(file) { |fp| copy(fp, mfp) }
+	if File.size(file) > 0x10000 # Bigass file, handle differently
+	  mfp.bigfile = file
+	  return mfp.close
+	else
+          return File.open(file) { |fp| sysrwloop(fp, mfp) }
+        end
       end
     end
   end
@@ -196,8 +224,43 @@ class MogileFS::MogileFS < MogileFS::Client
     res = @backend.rename :domain => @domain, :from_key => from, :to_key => to
 
     if res.nil? and @backend.lasterr != 'unknown_key' then
-      raise "unable to rename #{from_key} to #{to_key}: #{@backend.lasterr}"
+      raise "unable to rename #{from} to #{to}: #{@backend.lasterr}"
+    end
+  end
+
+  ##
+  # Returns the size of +key+.
+  def size(key)
+    paths = get_paths key
+
+    return nil unless paths
+
+    paths.each do |path|
+      next unless path
+      case path
+      when /^http:\/\// then
+        begin
+          url = URI.parse path
+
+          req = Net::HTTP::Head.new url.request_uri
+
+          res = timeout @get_file_data_timeout, MogileFS::Timeout do
+            Net::HTTP.start url.host, url.port do |http|
+              http.request req
+            end
+          end
+
+          return res['Content-Length'].to_i
+        rescue MogileFS::Timeout
+          next
+        end
+      else
+        next unless File.exist? path
+        return File.size(path)
+      end
     end
+
+    return nil
   end
 
   ##
@@ -215,19 +278,5 @@ class MogileFS::MogileFS < MogileFS::Client
     return keys, res['next_after']
   end
 
-  private
-
-  def copy(from, to) # HACK use FileUtils
-    bytes = 0
-
-    until from.eof? do
-      chunk = from.read 8192
-      to.write chunk
-      bytes += chunk.length
-    end
-
-    return bytes
-  end
-
 end
 

data/lib/mogilefs/util.rb

@@ -0,0 +1,30 @@
+module MogileFS::Util
+
+  CHUNK_SIZE = 65536
+
+  # for copying large files while avoiding GC thrashing as much as possible
+  def sysrwloop(io_rd, io_wr)
+    copied = 0
+    # avoid making sysread repeatedly allocate a new String
+    # This is not well-documented, but both read/sysread can take
+    # an optional second argument to use as the buffer to avoid
+    # GC overhead of creating new strings in a loop
+    buf = ' ' * CHUNK_SIZE # preallocate to avoid GC thrashing
+    io_wr.sync = true
+    loop do
+      begin
+        io_rd.sysread(CHUNK_SIZE, buf)
+        loop do
+          w = io_wr.syswrite(buf)
+          copied += w
+          break if w == buf.size
+          buf = buf[w..-1]
+        end
+      rescue EOFError
+        break
+      end
+    end
+    copied
+  end # sysrwloop
+
+end

data/test/setup.rb

@@ -1,5 +1,12 @@
 require 'test/unit'
 
+require 'fileutils'
+require 'tmpdir'
+require 'stringio'
+
+require 'rubygems'
+require 'test/zentest_assertions'
+
 $TESTING = true
 
 require 'mogilefs'
@@ -32,22 +39,114 @@ class FakeBackend
 
 end
 
+class FakeSocket
+
+  attr_reader :read_s
+  attr_reader :write_s
+  attr_reader :sync
+
+  def initialize(read = '', write = StringIO.new)
+    @read_s = read.class.method_defined?(:sysread) ? read : StringIO.new(read)
+    @write_s = write
+    @closed = false
+    @sync = false
+  end
+
+  def sync=(do_sync)
+    @write_s.sync = do_sync
+    @read_s.sync = do_sync
+  end
+
+  def closed?
+    @closed
+  end
+
+  def close
+    @closed = true
+    return nil
+  end
+
+  def gets
+    @read_s.gets
+  end
+
+  def peeraddr
+    ['AF_INET', 6001, 'localhost', '127.0.0.1']
+  end
+
+  def read(bytes)
+    @read_s.read bytes
+  end
+
+  def sysread(bytes, buf = '')
+    @read_s.sysread bytes, buf
+  end
+
+  def recv_nonblock(bytes, flags = 0)
+    ret = @read_s.sysread(bytes)
+    # Ruby doesn't expose pread(2)
+    if (flags & Socket::MSG_PEEK) != 0
+      @read_s.sysseek(-ret.size, IO::SEEK_CUR)
+    end
+    ret
+  end
+  alias_method :recv, :recv_nonblock
+
+  def write(data)
+    @write_s.write data
+  end
+
+  def syswrite(data)
+    @write_s.syswrite data
+  end
+
+end
+
 class MogileFS::Client
   attr_writer :readonly
 end
 
+class TCPSocket
+
+  class << self
+
+    attr_accessor :connections
+    attr_accessor :sockets
+
+    alias old_new new
+
+    def new(host, port)
+      raise Errno::ECONNREFUSED if @sockets.empty?
+      @connections << [host, port]
+      @sockets.pop
+    end
+
+    alias open new
+
+  end
+
+end
+
 class TestMogileFS < Test::Unit::TestCase
 
+  undef_method :default_test
+
   def setup
-    return if self.class == TestMogileFS
-    @root = '/mogilefs/test'
+    @tempdir = File.join Dir.tmpdir, "test_mogilefs_#{$$}"
+    @root = File.join @tempdir, 'root'
+    FileUtils.mkdir_p @root
+
     @client = @klass.new :hosts => ['kaa:6001'], :domain => 'test',
                                   :root => @root
     @backend = FakeBackend.new
     @client.instance_variable_set '@backend', @backend
+
+    TCPSocket.sockets = []
+    TCPSocket.connections = []
   end
 
-  def test_nothing
+  def teardown
+    FileUtils.rm_rf @tempdir
   end
 
 end

data/test/test_backend.rb

@@ -1,4 +1,5 @@
 require 'test/unit'
+require 'test/setup'
 
 $TESTING = true
 
@@ -12,30 +13,11 @@ class MogileFS::Backend
 
 end
 
-class FakeSocket
-
-  def initialize
-    @closed = false
-  end
-
-  def closed?
-    @closed
-  end
-
-  def close
-    @closed = true
-    return nil
-  end
-
-  def peeraddr
-    ['AF_INET', 6001, 'localhost', '127.0.0.1']
-  end
-
-end
-
 class TestBackend < Test::Unit::TestCase
 
   def setup
+    TCPSocket.connections = []
+    TCPSocket.sockets = []
     @backend = MogileFS::Backend.new :hosts => ['localhost:1']
   end
 

data/test/test_mogilefs.rb

@@ -1,15 +1,23 @@
 require 'test/setup'
+require 'stringio'
+require 'tempfile'
+require 'fileutils'
 
 class URI::HTTP
 
   class << self
     attr_accessor :read_data
+    attr_accessor :open_data
   end
 
   def read
     self.class.read_data.shift
   end
 
+  def open(&block)
+    yield self.class.open_data
+  end
+
 end
 
 class TestMogileFS__MogileFS < TestMogileFS
@@ -21,7 +29,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_initialize
     assert_equal 'test', @client.domain
-    assert_equal '/mogilefs/test', @client.root
+    assert_equal @root, @client.root
 
     assert_raises ArgumentError do
       MogileFS::MogileFS.new :hosts => ['kaa:6001'], :root => '/mogilefs/test'
@@ -39,6 +47,43 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 'data!', @client.get_file_data('key')
   end
 
+  def test_get_file_data_http_block
+    tmpfp = Tempfile.new('test_mogilefs.open_data')
+    nr = 100 # tested with 1000
+    chunk_size = 1024 * 1024
+    expect_size = nr * chunk_size
+    header = "HTTP/1.0 200 OK\r\n" \
+             "Content-Length: #{expect_size}\r\n\r\n"
+    assert_equal header.size, tmpfp.syswrite(header)
+    nr.times { assert_equal chunk_size, tmpfp.syswrite(' ' * chunk_size) }
+    assert_equal expect_size + header.size, File.size(tmpfp.path)
+    tmpfp.sysseek(0)
+    socket = FakeSocket.new(tmpfp)
+    TCPSocket.sockets << socket
+
+    path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
+    path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
+
+    @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
+
+    data = Tempfile.new('test_mogilefs.dest_data')
+    @client.get_file_data('key') do |fp|
+      buf = ''
+      read_nr = nr = 0
+      loop do
+        begin
+          fp.sysread(16384, buf)
+          read_nr = buf.size
+          nr += read_nr
+          assert_equal read_nr, data.syswrite(buf), "partial write"
+        rescue EOFError
+          break
+        end
+      end
+      assert_equal expect_size, nr, "size mismatch"
+    end
+  end
+
   def test_get_paths
     path1 = 'rur-1/dev1/0/000/000/0000000062.fid'
     path2 = 'rur-2/dev2/0/000/000/0000000062.fid'
@@ -114,8 +159,120 @@ class TestMogileFS__MogileFS < TestMogileFS
     end
   end
 
+  def test_size_http
+    socket = FakeSocket.new <<-EOF
+HTTP/1.0 200 OK\r
+Content-Length: 5\r
+    EOF
+
+    TCPSocket.sockets << socket
+
+    path = 'http://example.com/path'
+
+    @backend.get_paths = { 'paths' => 1, 'path1' => path }
+
+    assert_equal 5, @client.size('key')
+
+    socket.write_s.rewind
+
+    assert_equal "HEAD /path HTTP/1.1\r\n", socket.write_s.gets
+
+    assert_equal ['example.com', 80], TCPSocket.connections.shift
+    assert_empty TCPSocket.connections
+  end
+
+  def test_size_nfs
+    path = File.join @root, 'path'
+
+    File.open path, 'w' do |fp| fp.write 'data!' end
+
+    @backend.get_paths = { 'paths' => 1, 'path1' => 'path' }
+
+    assert_equal 5, @client.size('key')
+  end
+
+  def test_store_content_http
+    socket = FakeSocket.new 'HTTP/1.0 200 OK'
+
+    TCPSocket.sockets << socket
+
+    @backend.create_open = {
+      'devid' => '1',
+      'path' => 'http://example.com/path',
+    }
+
+    @client.store_content 'new_key', 'test', 'data'
+
+    expected = <<-EOF.chomp
+PUT /path HTTP/1.0\r
+Content-Length: 4\r
+\r
+data
+    EOF
+
+    assert_equal expected, socket.write_s.string
+
+    assert_equal ['example.com', 80], TCPSocket.connections.shift
+    assert_empty TCPSocket.connections
+  end
+
+  def test_store_content_http_empty
+    socket = FakeSocket.new 'HTTP/1.0 200 OK'
+
+    TCPSocket.sockets << socket
+
+    @backend.create_open = {
+      'devid' => '1',
+      'path' => 'http://example.com/path',
+    }
+
+    @client.store_content 'new_key', 'test', ''
+
+    expected = <<-EOF
+PUT /path HTTP/1.0\r
+Content-Length: 0\r
+\r
+    EOF
+
+    assert_equal expected, socket.write_s.string
+
+    assert_equal ['example.com', 80], TCPSocket.connections.shift
+    assert_empty TCPSocket.connections
+  end
+
+  def test_store_content_nfs
+    @backend.create_open = {
+      'dev_count' => '1',
+      'devid_1' => '1',
+      'path_1' => '/path',
+    }
+
+    @client.store_content 'new_key', 'test', 'data'
+
+    dest_file = File.join(@root, 'path')
+
+    assert File.exist?(dest_file)
+    assert_equal 'data', File.read(dest_file)
+  end
+
+  def test_store_content_nfs_empty
+    @backend.create_open = {
+      'dev_count' => '1',
+      'devid_1' => '1',
+      'path_1' => '/path',
+    }
+
+    @client.store_content 'new_key', 'test', ''
+
+    dest_file = File.join(@root, 'path')
+
+    assert File.exist?(dest_file)
+    assert_equal '', File.read(dest_file)
+  end
+
   def test_store_content_readonly
     @client.readonly = true
+
     assert_raises RuntimeError do
       @client.store_content 'new_key', 'test', nil
     end
@@ -130,23 +287,34 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_rename_existing
     @backend.rename = {}
-    assert_nothing_raised do
-      assert_equal(nil, @client.rename('from_key', 'to_key'))
-    end
+
+    assert_nil @client.rename('from_key', 'to_key')
   end
 
   def test_rename_nonexisting
     @backend.rename = 'unknown_key', ''
-    assert_nothing_raised do
-      assert_equal(nil, @client.rename('from_key', 'to_key'))
+
+    assert_nil @client.rename('from_key', 'to_key')
+  end
+
+  def test_rename_no_key
+    @backend.rename = 'no_key', ''
+
+    e = assert_raises RuntimeError do
+      @client.rename 'new_key', 'test'
     end
+
+    assert_equal 'unable to rename new_key to test: no_key', e.message
   end
 
   def test_rename_readonly
     @client.readonly = true
-    assert_raises RuntimeError do
+
+    e = assert_raises RuntimeError do
       @client.rename 'new_key', 'test'
     end
+
+    assert_equal 'readonly mogilefs', e.message
   end
 
   def test_sleep

metadata

@@ -1,45 +1,56 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.4.3
-specification_version: 1
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  version: 1.2.1
-date: 2007-07-31 00:00:00 -07:00
-summary: A Ruby MogileFS client
-require_paths: 
-- lib
-email: drbrain@segment7.net
-homepage: http://seattlerb.org/mogilefs-client
-rubyforge_project: seattlerb
-description: A Ruby MogileFS client.  MogileFS is a distributed filesystem written by Danga Interactive.  This client supports NFS and HTTP modes.
-autorequire: 
-default_executable: 
-bindir: bin
-has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
-  version: 
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
-  version: 
+  version: 1.3.0
 platform: ruby
-signing_key: 
-cert_chain: 
-post_install_message: 
 authors: 
+- Eric Wong
 - Eric Hodel
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-09-18 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: ZenTest
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 3.6.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.7.0
+    version: 
+description: A Ruby MogileFS client.  MogileFS is a distributed filesystem written by Danga Interactive.  This client supports NFS and HTTP modes.
+email: normalperson@yhbt.net
+executables: 
+- mog
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
 files: 
 - History.txt
 - LICENSE.txt
 - Manifest.txt
 - README.txt
 - Rakefile
+- bin/mog
 - lib/mogilefs.rb
 - lib/mogilefs/admin.rb
 - lib/mogilefs/backend.rb
@@ -48,39 +59,43 @@ files:
 - lib/mogilefs/mogilefs.rb
 - lib/mogilefs/nfsfile.rb
 - lib/mogilefs/pool.rb
+- lib/mogilefs/util.rb
 - test/setup.rb
 - test/test_admin.rb
 - test/test_backend.rb
 - test/test_client.rb
 - test/test_mogilefs.rb
 - test/test_pool.rb
-test_files: 
-- test/test_admin.rb
-- test/test_backend.rb
-- test/test_client.rb
-- test/test_mogilefs.rb
-- test/test_pool.rb
+has_rdoc: true
+homepage: http://seattlerb.org/mogilefs-client
+post_install_message: 
 rdoc_options: 
 - --main
 - README.txt
-extra_rdoc_files: 
-- History.txt
-- LICENSE.txt
-- Manifest.txt
-- README.txt
-executables: []
-
-extensions: []
-
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
 requirements: []
 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.2.2
-    version: 
+rubyforge_project: seattlerb
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A Ruby MogileFS client
+test_files: 
+- test/test_pool.rb
+- test/test_mogilefs.rb
+- test/test_client.rb
+- test/test_admin.rb
+- test/test_backend.rb