mogilefs-client 2.2.0 → 3.0.0.rc1

data/lib/mogilefs/mysql.rb

@@ -1,7 +1,6 @@
 # -*- encoding: binary -*-
-require 'mogilefs'
-require 'mogilefs/backend' # for the exceptions
-
+# Consider this deprecated, to be removed at some point...
+#
 # read-only interface that can be a backend for MogileFS::MogileFS
 #
 # This provides direct, read-only access to any slave MySQL database to
@@ -29,7 +28,7 @@ class MogileFS::Mysql
   ##
   # Lists keys starting with +prefix+ follwing +after+ up to +limit+.  If
   # +after+ is nil the list starts at the beginning.
-  def _list_keys(domain, prefix = '', after = '', limit = 1000, &block)
+  def _list_keys(domain, prefix = '', after = '', limit = 1000)
     # this code is based on server/lib/MogileFS/Worker/Query.pm
     dmid = get_dmid(domain)
 

data/lib/mogilefs/paths_size.rb

@@ -0,0 +1,24 @@
+# -*- encoding: binary -*-
+# This is only a hack for old MogileFS installs that didn't have file_info
+require "net/http"
+require "uri"
+module MogileFS::PathsSize
+  def self.call(paths)
+    errors = {}
+    paths.each do |path|
+      uri = URI.parse(path)
+      begin
+        case r = Net::HTTP.start(uri.host, uri.port) { |x| x.head(uri.path) }
+        when Net::HTTPOK
+          return r["Content-Length"].to_i
+        else
+          errors[path] = r
+        end
+      rescue => err
+        errors[path] = err
+      end
+    end
+    errors = errors.map { |path,err| "#{path} - #{err.message} (#{err.class})" }
+    raise MogileFS::Error, "all paths failed with HEAD: #{errors.join(', ')}"
+  end
+end

data/lib/mogilefs/pool.rb

@@ -1,6 +1,5 @@
 # -*- encoding: binary -*-
 require 'thread'
-require 'mogilefs'
 
 class MogileFS::Pool
 

data/lib/mogilefs/socket.rb

@@ -0,0 +1,9 @@
+# -*- encoding: binary -*-
+# internal implementation details here, do not rely on this in your code
+require "mogilefs/socket_common"
+begin
+  raise LoadError, "testing pure Ruby version" if ENV["MOGILEFS_CLIENT_PURE"]
+  require "mogilefs/socket/kgio"
+rescue LoadError
+  require "mogilefs/socket/pure_ruby"
+end

data/lib/mogilefs/socket/kgio.rb

@@ -0,0 +1,55 @@
+# -*- encoding: binary -*-
+# internal implementation details here, do not rely on them in your code
+require "kgio"
+
+class MogileFS::Socket < Kgio::Socket
+  include MogileFS::SocketCommon
+
+  def self.start(host, port)
+    sock = super(Socket.sockaddr_in(port, host))
+    sock.post_init(host, port)
+  end
+
+  def self.tcp(host, port, timeout = 5)
+    sock = start(host, port)
+    unless sock.kgio_wait_writable(timeout)
+      sock.close
+      raise MogileFS::Timeout, 'socket connect timeout'
+    end
+    sock
+  end
+
+  def timed_read(len, dst = "", timeout = 5)
+    case rc = kgio_tryread(len, dst)
+    when :wait_readable
+      kgio_wait_readable(timeout) or unreadable_socket!
+    else
+      return rc
+    end while true
+  end
+
+  def timed_peek(len, dst, timeout = 5)
+    case rc = kgio_trypeek(len, dst)
+    when :wait_readable
+      kgio_wait_readable(timeout) or unreadable_socket!
+    else
+      return rc
+    end while true
+  end
+
+  def timed_write(buf, timeout = 5)
+    written = 0
+    expect = buf.bytesize
+    case rc = kgio_trywrite(buf)
+    when :wait_writable
+      kgio_wait_writable(timeout) or request_truncated!(written, expect)
+    when String
+      written += expect - rc.bytesize
+      buf = rc
+    else
+      return expect
+    end while true
+  end
+
+  alias write timed_write
+end

data/lib/mogilefs/socket/pure_ruby.rb

@@ -0,0 +1,70 @@
+# -*- encoding: binary -*-
+# internal implementation details here, do not rely on them in your code
+
+class MogileFS::Socket < Socket
+  include MogileFS::SocketCommon
+
+  def self.start(host, port)
+    sock = new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+    begin
+      sock.connect_nonblock(sockaddr_in(port, host))
+    rescue Errno::EINPROGRESS
+    end
+    sock.post_init(host, port)
+  end
+
+  def self.tcp(host, port, timeout = 5)
+    sock = start(host, port)
+    unless IO.select(nil, [ sock ], nil, timeout)
+      sock.close
+      raise MogileFS::Timeout, 'socket connect timeout'
+    end
+    sock
+  end
+
+  def timed_read(len, dst = "", timeout = 5)
+    begin
+      IO.select([self], nil, nil, timeout) or unreadable_socket!
+      return read_nonblock(len, dst)
+    rescue Errno::EAGAIN
+    rescue EOFError
+      return
+    end while true
+  end
+
+  def timed_peek(len, dst, timeout = 5)
+    begin
+      IO.select([self], nil, nil, timeout) or unreadable_socket!
+      rc = recv_nonblock(len, Socket::MSG_PEEK)
+      return rc.empty? ? nil : dst.replace(rc)
+    rescue Errno::EAGAIN
+    rescue EOFError
+      dst.replace("")
+      return
+    end while true
+  end
+
+  def timed_write(buf, timeout = 5)
+    written = 0
+    expect = buf.bytesize
+    begin
+      rc = write_nonblock(buf)
+      return expect if rc == buf.bytesize
+      written += rc
+
+      if buf.respond_to?(:byteslice)
+        buf = buf.byteslice(rc, buf.bytesize)
+      else
+        if buf.respond_to?(:encoding) && buf.encoding != Encoding::BINARY
+          buf = buf.dup.force_encoding(Encoding::BINARY)
+        end
+        buf = buf.slice(rc, buf.bytesize)
+      end
+    rescue Errno::EAGAIN
+      IO.select(nil, [self], nil, timeout) or
+        request_truncated!(written, expect)
+    end while true
+  end
+
+  alias write timed_write
+end

data/lib/mogilefs/socket_common.rb

@@ -0,0 +1,58 @@
+# -*- encoding: binary -*-
+# internal implementation details here, do not rely on this in your code
+require "socket"
+
+module MogileFS::SocketCommon
+  attr_reader :mogilefs_addr
+
+  def post_init(host, port)
+    @mogilefs_addr = "#{host}:#{port}"
+    Socket.const_defined?(:TCP_NODELAY) and
+      setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+    self
+  end
+
+  def unreadable_socket!
+    raise MogileFS::UnreadableSocketError,
+          "#@mogilefs_addr never became readable"
+  end
+
+  def request_truncated!(written, expect)
+    raise MogileFS::RequestTruncatedError,
+          "request truncated (sent #{written} expected #{expect})"
+  end
+
+  SEP_RE = /\A(.*?#{Regexp.escape("\n")})/
+  def timed_gets(timeout = 5)
+    unless defined?(@rbuf) && @rbuf
+      @rbuf = timed_read(1024, "", timeout) or return # EOF
+    end
+    begin
+      @rbuf.sub!(SEP_RE, "") and return $1
+      tmp ||= ""
+      if timed_read(1024, tmp, timeout)
+        @rbuf << tmp
+      else
+        # EOF, return the last buffered bit even without SEP_RE matching
+        # (not ideal for MogileFS, this is an error)
+        return @rbuf.empty? ? nil : @rbuf.slice!(0, @rbuf.size)
+      end
+    end while true
+  end
+
+  def read(size, buf = "", timeout = 5)
+    timed_read(size, buf, timeout) or return # nil/EOF
+
+    while size > buf.bytesize
+      tmp ||= ""
+      timed_read(size - buf.bytesize, tmp, timeout) or return buf # truncated
+      buf << tmp
+    end
+
+    buf # full read
+  end
+
+  def readpartial(size, buf = "", timeout = 5)
+    timed_read(size, buf, timeout) or raise EOFError, "end of file reached"
+  end
+end

data/lib/mogilefs/util.rb

@@ -1,96 +1,13 @@
 # -*- encoding: binary -*-
-require 'mogilefs'
-require 'socket'
 
 module MogileFS::Util
 
-  CHUNK_SIZE = 65536
-
-  # for copying large files while avoiding GC thrashing as much as possible
-  # writes the contents of io_rd into io_wr, running through filter if
-  # it is a Proc object.  The filter proc must respond to a string
-  # argument (and return a string) and to nil (possibly returning a
-  # string or nil).  This can be used to filter I/O through an
-  # Zlib::Inflate or Digest::MD5 object
-  def sysrwloop(io_rd, io_wr, filter = nil)
-    copied = 0
-    # avoid making sysread repeatedly allocate a new String
-    # This is not well-documented, but both read/sysread can take
-    # an optional second argument to use as the buffer to avoid
-    # GC overhead of creating new strings in a loop
-    buf = ' ' * CHUNK_SIZE # preallocate to avoid GC thrashing
-    io_rd.flush rescue nil # flush may be needed for sockets/pipes, be safe
-    io_wr.flush
-    io_rd.sync = io_wr.sync = true
-    loop do
-      b = begin
-        io_rd.sysread(CHUNK_SIZE, buf)
-      rescue Errno::EAGAIN, Errno::EINTR
-        IO.select([io_rd], nil, nil, nil)
-        retry
-      rescue EOFError
-        break
-      end
-      b = filter.call(b) if filter
-      copied += syswrite_full(io_wr, b)
-    end
-
-    # filter must take nil as a possible argument to indicate EOF
-    if filter
-      b = filter.call(nil)
-      copied += syswrite_full(io_wr, b) if b && b.length > 0
-    end
-    copied
-  end # sysrwloop
-
-  # writes the contents of buf to io_wr in full w/o blocking
-  def syswrite_full(io_wr, buf, timeout = nil)
-    written = 0
-    loop do
-      begin
-        w = io_wr.syswrite(buf)
-        written += w
-        return written if w == buf.size
-        buf = buf[w..-1]
-
-        # a short syswrite means the next syswrite will likely block
-        # inside the interpreter.  so force an IO.select on it so we can
-        # timeout there if one was specified
-        raise Errno::EAGAIN if timeout
-      rescue Errno::EAGAIN, Errno::EINTR
-        t0 = Time.now if timeout
-        IO.select(nil, [io_wr], nil, timeout)
-        if timeout && ((timeout -= (Time.now - t0)) < 0)
-          raise MogileFS::Timeout, 'syswrite_full timeout'
-        end
-      end
-    end
-    # should never get here
-  end
-
-  def sysread_full(io_rd, size, timeout = nil, full_timeout = false)
-    tmp = [] # avoid expensive string concatenation with every loop iteration
-    reader = io_rd.method(timeout ? :read_nonblock : :sysread)
-    begin
-      while size > 0
-        tmp << reader.call(size)
-        size -= tmp.last.size
-      end
-    rescue Errno::EAGAIN, Errno::EINTR
-      t0 = Time.now
-      r = IO.select([ io_rd ], nil, nil, timeout)
-      if timeout
-        timeout -= (Time.now - t0) if full_timeout
-        if !(r && r[0]) || timeout < 0
-          raise MogileFS::Timeout, 'sysread_full timeout'
-        end
-      end
-      retry
-    rescue EOFError
-    end
-    tmp.join('')
-  end
-
+  # MogileFS::Util::StoreContent allows you to roll your own method
+  # of streaming data on an upload (instead of using a string or file)
+  #
+  # Current versions of this library support streaming a IO or IO-like
+  # object to using MogileFS::MogileFS#store_file, so using StoreContent
+  # may no longer be necessary.
   class StoreContent < Proc
     def initialize(total_size, &writer_proc)
       @total_size = total_size
@@ -100,7 +17,6 @@ module MogileFS::Util
       @total_size
     end
   end
-
 end
 
 require 'timeout'
@@ -109,82 +25,3 @@ require 'timeout'
 # reason we require the 'timeout' module, otherwise that module is
 # broken and worthless to us.
 class MogileFS::Timeout < Timeout::Error; end
-
-class Socket
-  attr_accessor :mogilefs_addr, :mogilefs_connected, :mogilefs_size
-
-  # Socket lacks peeraddr method of the IPSocket/TCPSocket classes
-  def mogilefs_peername
-    Socket.unpack_sockaddr_in(getpeername).reverse.map {|x| x.to_s }.join(':')
-  end
-
-  def mogilefs_init(host = nil, port = nil)
-    return true if defined?(@mogilefs_connected)
-
-    @mogilefs_addr = Socket.sockaddr_in(port, host).freeze if port && host
-
-    begin
-      connect_nonblock(@mogilefs_addr)
-      @mogilefs_connected = true
-    rescue Errno::EINPROGRESS
-      nil
-    rescue Errno::EISCONN
-      @mogilefs_connected = true
-    end
-  end
-
-  class << self
-
-    # Creates a new (TCP) Socket and initiates (but does not wait for) the
-    # connection
-    def mogilefs_new_nonblock(host, port)
-      sock = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-      if defined?(Socket::TCP_NODELAY)
-        sock.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
-      end
-      sock.mogilefs_init(host, port)
-      sock
-    end
-
-    # Like TCPSocket.new(host, port), but with an explicit timeout
-    # (and we don't care for local address/port we're binding to).
-    # This raises MogileFS::Timeout if timeout expires
-    def mogilefs_new(host, port, timeout = 5.0)
-      sock = mogilefs_new_nonblock(host, port) or return sock
-
-      while timeout > 0
-        t0 = Time.now
-        r = IO.select(nil, [sock], nil, timeout)
-        return sock if r && r[1] && sock.mogilefs_init
-        timeout -= (Time.now - t0)
-      end
-
-      sock.close rescue nil
-      raise MogileFS::Timeout, 'socket write timeout'
-    end
-
-    include MogileFS::Util
-
-    # Makes a request on a new TCP Socket and returns with a readble socket
-    # within the given timeout.
-    # This raises MogileFS::Timeout if timeout expires
-    def mogilefs_new_request(host, port, request, timeout = 5.0)
-      t0 = Time.now
-      sock = mogilefs_new(host, port, timeout)
-      syswrite_full(sock, request, timeout)
-      timeout -= (Time.now - t0)
-      if timeout < 0
-        sock.close rescue nil
-        raise MogileFS::Timeout, 'socket read timeout'
-      end
-      r = IO.select([sock], nil, nil, timeout)
-      return sock if r && r[0]
-
-      sock.close rescue nil
-      raise MogileFS::Timeout, 'socket read timeout'
-    end
-
-  end
-
-end
-

data/test/aggregate.rb

@@ -1,14 +1,14 @@
-#!/usr/bin/ruby -n
+#!/usr/bin/ruby
 # -*- encoding: binary -*-
-BEGIN { $tests = $assertions = $failures = $errors = 0 }
+$tests = $assertions = $failures = $errors = 0
 
-$_ =~ /(\d+) tests, (\d+) assertions, (\d+) failures, (\d+) errors/ or next
-$tests += $1.to_i
-$assertions += $2.to_i
-$failures += $3.to_i
-$errors += $4.to_i
+STDIN.each_line do |l|
+  l =~ /(\d+) tests, (\d+) assertions, (\d+) failures, (\d+) errors/ or next
+  $tests += $1.to_i
+  $assertions += $2.to_i
+  $failures += $3.to_i
+  $errors += $4.to_i
+end
 
-END {
-  printf("\n%d tests, %d assertions, %d failures, %d errors\n",
-         $tests, $assertions, $failures, $errors)
-}
+printf("\n%d tests, %d assertions, %d failures, %d errors\n",
+       $tests, $assertions, $failures, $errors)