mogilefs-client 3.0.0.rc1 → 3.0.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 CONSTANT = "MogileFS::VERSION"
 RVF = "lib/mogilefs/version.rb"
-DEF_VER = "v3.0.0-rc1"
+DEF_VER = "v3.0.0"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/examples/mogstored_rack.rb

@@ -1,5 +1,4 @@
 # -*- encoding: binary -*-
-require 'tempfile'
 require 'digest/md5'
 require 'rack'
 
@@ -22,11 +21,28 @@ class MogstoredRack
 
   def initialize(root, opts = {})
     @root = File.expand_path(root)
+    @io_size = opts[:io_size] || 0x100000
     @rack_file = (opts[:app] || Rack::File.new(@root))
     @fsync = !! opts[:fsync]
     @creat_perms = opts[:creat_perms] || (~File.umask & 0666)
     @mkdir_perms = opts[:mkdir_perms] || (~File.umask & 0777)
     @reread_verify = !! opts[:reread_verify]
+    @open_flags = opts[:open_flags] || 0
+    @open_flags |= IO::RDWR | IO::CREAT | IO::EXCL
+  end
+
+  def tmpfile(basename, dir)
+    t = Time.now.utc.strftime("%Y%m%d%H%M%S")
+    seq = 0
+    begin
+      fp = File.open("#{dir}/#{basename}.#{t}.#{seq}.tmp", @open_flags, 0600)
+    rescue Errno::EEXIST
+      retry if (seq += 1) < 10
+      raise
+    end
+    fp.binmode
+    fp.sync = true
+    fp
   end
 
   def call(env)
@@ -75,32 +91,29 @@ class MogstoredRack
     dir = File.dirname(path)
     File.directory?(dir) or return r(403)
 
-    Tempfile.open([File.basename(path), ".tmp"], dir) do |tmp|
-      tmp = tmp.to_io # delegated method calls are slower
-      tmp.sync = true
-      tmp.binmode
-      buf = ""
-      received = put_loop(env["rack.input"], tmp, buf)
-      err = content_md5_fail?(env, received) and return err
-      if @reread_verify && err = reread_md5_fail?(env, tmp, received, buf)
-        return err
-      end
-      tmp.chmod(@creat_perms)
-      begin
-        File.link(tmp.path, path)
-      rescue Errno::EEXIST
-        err = rename_overwrite_fail?(tmp.path, path) and return err
-      end
-      fsync(dir, tmp) if @fsync
-      resp = r(201)
-      resp[1]["X-Received-Content-MD5"] = received
-      return resp
+    tmp = tmpfile(File.basename(path), dir)
+    buf = ""
+    received = put_loop(env["rack.input"], tmp, buf)
+    err = content_md5_fail?(env, received) and return err
+    if @reread_verify && err = reread_md5_fail?(env, tmp, received, buf)
+      return err
     end
+    tmp.chmod(@creat_perms)
+    File.rename(tmp.path, path)
+    fsync(dir, tmp) if @fsync
+    resp = r(201)
+    resp[1]["X-Received-Content-MD5"] = received
+    resp
+    rescue
+      File.unlink(tmp.path) if tmp && File.exist?(tmp.path)
+      raise
+    ensure
+      tmp.close if tmp
   end
 
   def put_loop(src, dst, buf)
     md5 = ContentMD5.new
-    while src.read(0x4000, buf)
+    while src.read(@io_size, buf)
       md5.update(buf)
       dst.write(buf)
     end
@@ -167,20 +180,6 @@ class MogstoredRack
            "received: #{received}", env)
   end
 
-  def rename_overwrite_fail?(src, dst)
-    10.times do
-      begin
-        tmp_dst = "#{dst}.#{rand}"
-        File.link(src, tmp_dst)
-      rescue Errno::EEXIST
-        next
-      end
-      File.rename(tmp_dst, dst)
-      return false # success!
-    end
-    r(409)
-  end
-
   # fsync each and every directory component above us on the same device
   def fsync(dir, tmp)
     tmp.fsync

data/lib/mogilefs/backend.rb

@@ -295,7 +295,10 @@ class MogileFS::Backend
   # Turns the +line+ response from the server into a Hash of options, an
   # error, or raises, as appropriate.
   def parse_response(line, request = nil)
-    if line =~ /^ERR\s+(\w+)\s*([^\r\n]*)/
+    case line
+    when /\AOK\s+\d*\s*(\S*)\r?\n\z/
+      url_decode($1)
+    when /\AERR\s+(\w+)\s*([^\r\n]*)/
       @lasterr = $1
       @lasterrstr = $2 ? url_unescape($2) : nil
       if request
@@ -304,12 +307,10 @@ class MogileFS::Backend
         return error(@lasterr).new(@lasterrstr)
       end
       raise error(@lasterr).new(@lasterrstr)
+    else
+      raise MogileFS::InvalidResponseError,
+            "Invalid response from server: #{line.inspect}"
     end
-
-    return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)\r\n\z/
-
-    raise MogileFS::InvalidResponseError,
-          "Invalid response from server: #{line.inspect}"
   end
 
   # this command is special since the cache is per-tracker, so we connect

data/lib/mogilefs/copy_stream.rb

@@ -3,7 +3,7 @@
 # internal compatibility class for older Rubies
 module MogileFS::CopyStream # :nodoc:
   @r_args = IO::RDONLY | IO::NOCTTY
-  @w_args = [ IO::WRONLY|IO::CREAT|IO::NOCTTY|IO::TRUNC, 0600 ]
+  @w_args = [ IO::WRONLY|IO::CREAT|IO::NOCTTY|IO::TRUNC, 0666 ]
   def self.copy_stream(src, dst)
     src_io = src.respond_to?(:to_str) ? File.open(src, @r_args) : src
     dst_io = dst.respond_to?(:to_str) ? File.open(dst, *@w_args) : dst

data/lib/mogilefs/http_file.rb

@@ -20,12 +20,27 @@ class MogileFS::HTTPFile < StringIO
   end
   class NonRetryableError < MogileFS::Error; end
 
+  class HTTPSock < MogileFS::Socket
+    attr_accessor :start
+
+    # Increase timeout as we become more invested in uploading with
+    # this socket.  The server could be experiencing I/O delays
+    # from large uploads because the sysadmin forgot to tune the
+    # VM sysctls for handling large files.
+    def write(buf)
+      timed_write(buf, Time.now - @start + 5.0)
+    end
+  end
+
   # :stopdoc:
   MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
   class << self
     attr_accessor :response_timeout_cb
   end
 
+  # temporary directories (nginx) may not be configured on the
+  # same device, necessitating a time-consuming full file copy
+  # instead of a quick rename(2)/link(2) operation
   @response_timeout_cb = lambda do |elapsed_time, bytes_uploaded|
     mbytes_uploaded = bytes_uploaded / (1024.0 * 1024.0)
     # assumes worst case is 10M/s on the remote storage disk
@@ -107,7 +122,8 @@ class MogileFS::HTTPFile < StringIO
   # returns file size if the socket finished writing
   def upload(devid, uri) # :nodoc:
     start = Time.now
-    sock = MogileFS::Socket.tcp(uri.host, uri.port)
+    sock = HTTPSock.tcp(uri.host, uri.port)
+    sock.start = start
     file_size = length
 
     if @streaming_io

data/lib/mogilefs/socket/kgio.rb

@@ -50,6 +50,4 @@ class MogileFS::Socket < Kgio::Socket
       return expect
     end while true
   end
-
-  alias write timed_write
 end

data/lib/mogilefs/socket/pure_ruby.rb

@@ -65,6 +65,4 @@ class MogileFS::Socket < Socket
         request_truncated!(written, expect)
     end while true
   end
-
-  alias write timed_write
 end

data/test/test_mogilefs.rb

@@ -249,7 +249,7 @@ class TestMogileFS__MogileFS < TestMogileFS
   end
 
   def test_store_content_http
-    received = Tempfile.new('recieved')
+    received = Tempfile.new('received')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
     t = TempServer.new(Proc.new do |serv, accept|
@@ -282,7 +282,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
 
   def test_store_content_with_writer_callback
-    received = Tempfile.new('recieved')
+    received = Tempfile.new('received')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 40\r\n\r\n"
     10.times do
       expected += "data"

data/test/test_mogilefs_integration.rb

@@ -19,8 +19,6 @@ class TestMogileFSIntegration < TestMogIntegration
     tmp.close!
     assert_equal 4, @client.get_file_data("CRUD", tmp_path)
     assert_equal "DATA", File.read(tmp_path)
-    st = File.stat(tmp_path)
-    assert_equal 0100600, st.mode
     File.unlink(tmp_path)
 
     sio = StringIO.new("")

metadata

@@ -1,15 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  hash: 15424119
-  prerelease: 6
+  hash: 7
+  prerelease: 
   segments: 
   - 3
   - 0
   - 0
-  - rc
-  - 1
-  version: 3.0.0.rc1
+  version: 3.0.0
 platform: ruby
 authors: 
 - Eric Wong
@@ -17,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-21 00:00:00 Z
+date: 2011-11-28 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -130,14 +128,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version 
-      hash: 25
+      hash: 3
       segments: 
-      - 1
-      - 3
-      - 1
-      version: 1.3.1
+      - 0
+      version: "0"
 requirements: []
 
 rubyforge_project: seattlerb