mogilefs-client 2.1.0 → 2.2.0

data/GNUmakefile

@@ -12,15 +12,15 @@ clean:
 
 
 ifndef V
-	quiet_pre = @echo '* $@';
-	quiet_post = >$(t) 2>&1
+  quiet_pre = @echo '* $@';
+  quiet_post = >$(t) 2>&1
 else
-	# we can't rely on -o pipefail outside of bash 3+,
-	# so we use a stamp file to indicate success and
-	# have rm fail if the stamp didn't get created
-	stamp = $@$(log_suffix).ok
-	quiet_pre = @echo $(ruby) $@ $(TEST_OPTS); ! test -f $(stamp) && (
-	quiet_post = && > $(stamp) )>&2 | tee $(t); rm $(stamp) 2>/dev/null
+  # we can't rely on -o pipefail outside of bash 3+,
+  # so we use a stamp file to indicate success and
+  # have rm fail if the stamp didn't get created
+  stamp = $@$(log_suffix).ok
+  quiet_pre = @echo $(ruby) $@ $(TEST_OPTS); ! test -f $(stamp) && (
+  quiet_post = && > $(stamp) )>&2 | tee $(t); rm $(stamp) 2>/dev/null
 endif
 ruby = ruby
 run_test = $(quiet_pre) setsid $(ruby) -w $@ $(TEST_OPTS) $(quiet_post) || \

data/History.txt

@@ -1,3 +1,7 @@
+= 2.2.0
+* internal cleanups (no public API breakage)
+* refactor backend socket/connection handling for reliability
+
 = 2.1.0
 * MySQL interface returns integer length and devcount (API change)
 * Ensure store_{content,file} always returns size (API fix)

data/lib/mogilefs.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 ##
 # MogileFS is a Ruby client for Danga Interactive's open source distributed
 # filesystem.
@@ -6,7 +7,7 @@
 
 module MogileFS
 
-  VERSION = '2.1.0'.freeze
+  VERSION = '2.2.0'
 
   ##
   # Raised when a socket remains unreadable for too long.

data/lib/mogilefs/admin.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'mogilefs/client'
 
 ##

data/lib/mogilefs/backend.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'mogilefs'
 require 'mogilefs/util'
 require 'thread'
@@ -74,10 +75,7 @@ class MogileFS::Backend
   # Closes this backend's socket.
 
   def shutdown
-    if @socket
-      @socket.close rescue nil # ignore errors
-      @socket = nil
-    end
+    @mutex.synchronize { shutdown_unlocked }
   end
 
   # MogileFS::MogileFS commands
@@ -147,29 +145,44 @@ class MogileFS::Backend
 
   private unless defined? $TESTING
 
+  # record-separator for mogilefsd responses, update this if the protocol
+  # changes
+  RS = "\n"
+
+  def shutdown_unlocked # :nodoc:
+    if @socket
+      @socket.close rescue nil # ignore errors
+      @socket = nil
+    end
+  end
+
   ##
   # Performs the +cmd+ request with +args+.
 
   def do_request(cmd, args)
+    response = nil
+    request = make_request cmd, args
     @mutex.synchronize do
-      request = make_request cmd, args
-
       begin
-        bytes_sent = socket.send request, 0
-      rescue SystemCallError
-        shutdown
-        raise MogileFS::UnreachableBackendError
-      end
-
-      unless bytes_sent == request.length then
-        raise MogileFS::RequestTruncatedError,
-          "request truncated (sent #{bytes_sent} expected #{request.length})"
+        io = socket
+        begin
+          bytes_sent = io.write request
+          bytes_sent == request.size or
+            raise MogileFS::RequestTruncatedError,
+               "request truncated (sent #{bytes_sent} expected #{request.size})"
+        rescue SystemCallError
+          raise MogileFS::UnreachableBackendError
+        end
+
+        readable?(io)
+        response = io.gets(RS) and return parse_response(response)
+      ensure
+        # we DO NOT want the response we timed out waiting for, to crop up later
+        # on, on the same socket, intersperesed with a subsequent request!
+        # we close the socket if it times out like this
+        response or shutdown_unlocked
       end
-
-      readable?
-
-      parse_response(socket.gets)
-    end
+    end # @mutex.synchronize
   end
 
   ##
@@ -192,11 +205,10 @@ class MogileFS::Backend
   # error, or raises, as appropriate.
 
   def parse_response(line)
-    if line =~ /^ERR\s+(\w+)\s*(.*)/ then
+    if line =~ /^ERR\s+(\w+)\s*([^\r\n]*)/
       @lasterr = $1
       @lasterrstr = $2 ? url_unescape($2) : nil
       raise error(@lasterr), @lasterrstr
-      return nil
     end
 
     return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)/
@@ -208,26 +220,18 @@ class MogileFS::Backend
   ##
   # Raises if the socket does not become readable in +@timeout+ seconds.
 
-  def readable?
+  def readable?(io = @socket)
     timeleft = @timeout
     peer = nil
     loop do
       t0 = Time.now
-      found = IO.select([socket], nil, nil, timeleft)
+      found = IO.select([io], nil, nil, timeleft)
       return true if found && found[0]
       timeleft -= (Time.now - t0)
+      timeleft >= 0 and next
+      peer = io ? "#{io.mogilefs_peername} " : nil
 
-      if timeleft < 0
-        peer = @socket ? "#{@socket.mogilefs_peername} " : nil
-
-        # we DO NOT want the response we timed out waiting for, to crop up later
-        # on, on the same socket, intersperesed with a subsequent request! so,
-        # we close the socket if it times out like this
-        shutdown
-        raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
-        break
-      end
-      shutdown
+      raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
     end
     false
   end

data/lib/mogilefs/bigfile.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'zlib'
 require 'digest/md5'
 require 'uri'

data/lib/mogilefs/client.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'mogilefs/backend'
 
 ##

data/lib/mogilefs/httpfile.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'stringio'
 require 'uri'
 require 'mogilefs/backend'
@@ -89,7 +90,6 @@ class MogileFS::HTTPFile < StringIO
   def upload(devid, uri)
     file_size = length
     sock = Socket.mogilefs_new(uri.host, uri.port)
-    sock.mogilefs_tcp_cork = true
 
     if @streaming_io
       file_size = @streaming_io.length
@@ -102,7 +102,6 @@ class MogileFS::HTTPFile < StringIO
       # Don't try to run out of memory
       File.open(@big_io, "rb") do |fp|
         file_size = fp.stat.size
-        fp.sync = true
         syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
                             "Content-Length: #{file_size}\r\n\r\n")
         sysrwloop(fp, sock)
@@ -111,7 +110,6 @@ class MogileFS::HTTPFile < StringIO
       syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
                           "Content-Length: #{length}\r\n\r\n#{string}")
     end
-    sock.mogilefs_tcp_cork = false
 
     line = sock.gets or
       raise EmptyResponseError, 'Unable to read response line from server'

data/lib/mogilefs/mogilefs.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'mogilefs/client'
 require 'mogilefs/util'
 

data/lib/mogilefs/mysql.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'mogilefs'
 require 'mogilefs/backend' # for the exceptions
 

data/lib/mogilefs/network.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'mogilefs'
 require 'socket'
 require 'mogilefs/util'

data/lib/mogilefs/pool.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'thread'
 require 'mogilefs'
 

data/lib/mogilefs/util.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'mogilefs'
 require 'socket'
 
@@ -112,15 +113,6 @@ class MogileFS::Timeout < Timeout::Error; end
 class Socket
   attr_accessor :mogilefs_addr, :mogilefs_connected, :mogilefs_size
 
-  TCP_CORK = 3 if ! defined?(TCP_CORK) && RUBY_PLATFORM =~ /linux/
-
-  def mogilefs_tcp_cork=(set)
-    if defined?(TCP_CORK)
-      self.setsockopt(SOL_TCP, TCP_CORK, set ? 1 : 0) rescue nil
-    end
-    set
-  end
-
   # Socket lacks peeraddr method of the IPSocket/TCPSocket classes
   def mogilefs_peername
     Socket.unpack_sockaddr_in(getpeername).reverse.map {|x| x.to_s }.join(':')
@@ -147,7 +139,6 @@ class Socket
     # connection
     def mogilefs_new_nonblock(host, port)
       sock = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-      sock.sync = true
       if defined?(Socket::TCP_NODELAY)
         sock.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
       end

data/setup.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 #
 # setup.rb
 #

data/test/aggregate.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/ruby -n
+# -*- encoding: binary -*-
 BEGIN { $tests = $assertions = $failures = $errors = 0 }
 
 $_ =~ /(\d+) tests, (\d+) assertions, (\d+) failures, (\d+) errors/ or next

data/test/setup.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 STDIN.sync = STDOUT.sync = STDERR.sync = true
 require 'test/unit'
 require 'tempfile'

data/test/test_admin.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/setup'
 
 class TestMogileFS__Admin < TestMogileFS

data/test/test_backend.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/unit'
 require 'test/setup'
 
@@ -57,7 +58,7 @@ class TestBackend < Test::Unit::TestCase
     socket_request = ''
     socket = Object.new
     def socket.closed?() false end
-    def socket.send(request, flags) raise SystemCallError, 'dummy' end
+    def socket.write(request) raise SystemCallError, 'dummy' end
 
     @backend.instance_variable_set '@socket', socket
 
@@ -92,7 +93,7 @@ class TestBackend < Test::Unit::TestCase
     socket_request = ''
     socket = Object.new
     def socket.closed?() false end
-    def socket.send(request, flags) return request.length - 1 end
+    def socket.write(request) return request.length - 1 end
 
     @backend.instance_variable_set '@socket', socket
 
@@ -127,21 +128,16 @@ class TestBackend < Test::Unit::TestCase
     end
   end
 
-  def test_readable_eh_readable
-    accept = Tempfile.new('accept')
-    tmp = TempServer.new(Proc.new do |serv, port|
-      client, client_addr = serv.accept
-      client.sync = true
-      accept.syswrite('.')
-      client.send('.', 0)
-      sleep
-    end)
+  def test_parse_response_newline
+    begin
+      @backend.parse_response("ERR you totally suck\r\n")
+    rescue MogileFS::Error => err
+      assert_equal 'MogileFS::Backend::YouError', err.class.to_s
+      assert_equal 'totally suck', err.message
+    end
 
-    @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
-    assert_equal true, @backend.readable?
-    assert_equal 1, accept.stat.size
-    ensure
-      TempServer.destroy_all!
+    assert_equal 'you', @backend.lasterr
+    assert_equal 'totally suck', @backend.lasterrstr
   end
 
   def test_readable_eh_not_readable
@@ -149,7 +145,7 @@ class TestBackend < Test::Unit::TestCase
     @backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{tmp.port}" ],
                                      :timeout => 0.5)
     begin
-      @backend.readable?
+      @backend.do_request 'foo', {}
     rescue MogileFS::UnreadableSocketError => e
       assert_equal "127.0.0.1:#{tmp.port} never became readable", e.message
     rescue Exception => err

data/test/test_bigfile.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/setup'
 require 'mogilefs/bigfile'
 

data/test/test_client.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/unit'
 require 'mogilefs'
 

data/test/test_db_backend.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/setup'
 require 'mogilefs/mysql'
 

data/test/test_mogilefs.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/setup'
 require 'stringio'
 require 'tempfile'

data/test/test_mysql.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/setup'
 require 'mogilefs/mysql'
 

data/test/test_network.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/setup'
 require 'mogilefs'
 require 'mogilefs/network'

data/test/test_pool.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/unit'
 
 $TESTING = true

data/test/test_util.rb

@@ -1,3 +1,4 @@
+# -*- encoding: binary -*-
 require 'test/setup'
 
 class TestMogileFS__Util < Test::Unit::TestCase

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  version: 2.1.0
+  hash: 7
+  prerelease: false
+  segments: 
+  - 2
+  - 2
+  - 0
+  version: 2.2.0
 platform: ruby
 authors: 
 - Eric Wong
@@ -10,29 +16,41 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-11 00:00:00 -07:00
+date: 2011-01-13 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: ZenTest
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 3
+        - 6
+        - 1
         version: 3.6.1
-    version: 
+  type: :development
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: hoe
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.10.0
-    version: 
+        hash: 47
+        segments: 
+        - 2
+        - 8
+        - 0
+        version: 2.8.0
+  type: :development
+  version_requirements: *id002
 description: "Source repository (git):"
 email: normalperson@yhbt.net
 executables: 
@@ -80,6 +98,8 @@ files:
 - test/test_util.rb
 has_rdoc: true
 homepage: http://seattlerb.rubyforge.org/mogilefs-client
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --main
@@ -87,32 +107,38 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: seattlerb
-rubygems_version: 1.3.1
+rubygems_version: 1.3.7
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: A Ruby MogileFS client
 test_files: 
-- test/test_bigfile.rb
-- test/test_pool.rb
 - test/test_db_backend.rb
-- test/test_mogilefs.rb
 - test/test_network.rb
 - test/test_mysql.rb
-- test/test_client.rb
-- test/test_admin.rb
 - test/test_util.rb
 - test/test_backend.rb
+- test/test_mogilefs.rb
+- test/test_client.rb
+- test/test_admin.rb
+- test/test_pool.rb
+- test/test_bigfile.rb