moneta 1.0.0 → 1.5.1

data/lib/moneta/server.rb

@@ -4,14 +4,171 @@ module Moneta
   # Moneta server to be used together with Moneta::Adapters::Client
   # @api public
   class Server
+    include Config
+
+    config :timeout, default: 1
+    config :max_size, default: 0x100000
+
+    # @api private
+    class Connection
+      def initialize(io, store, max_size)
+        @io = io
+        @store = store
+        @max_size = max_size
+        @fiber = Fiber.new { run }
+      end
+
+      def resume(result = nil)
+        @fiber.resume result
+      end
+
+      private
+
+      # The return value of this function will be sent to the reactor.
+      #
+      # @return [:closed,Exception]
+      def run
+        catch :closed do
+          loop { write_dispatch(read_msg) }
+        end
+        :closed
+      rescue => ex
+        ex
+      ensure
+        @io.close unless @io.closed?
+      end
+
+      def dispatch(method, args)
+        case method
+        when :key?, :load, :delete, :increment, :create, :features
+          @store.public_send(method, *args)
+        when :store, :clear
+          @store.public_send(method, *args)
+          nil
+        when :each_key
+          yield_each(@store.each_key)
+          nil
+        end
+      rescue => ex
+        ex
+      end
+
+      def write_dispatch(msg)
+        method, *args = msg
+        result = dispatch(method, args)
+        write(result)
+      end
+
+      def read_msg
+        size = read(4).unpack('N').first
+        throw :closed, 'Message too big' if size > @max_size
+        Marshal.load(read(size))
+      end
+
+      def read(len)
+        buffer = ''
+        loop do
+          begin
+            case received = @io.recv_nonblock(len)
+            when '', nil
+              throw :closed, 'Closed during read'
+            else
+              buffer << received
+              len -= received.bytesize
+            end
+          rescue IO::WaitReadable, IO::WaitWritable
+            yield_to_reactor(:read)
+          rescue Errno::ECONNRESET
+            throw :closed, 'Closed during read'
+          rescue IOError => ex
+            if ex.message =~ /closed stream/
+              throw :closed, 'Closed during read'
+            else
+              raise
+            end
+          end
+          break if len == 0
+        end
+        buffer
+      end
+
+      def write(obj)
+        buffer = pack(obj)
+        until buffer.empty?
+          begin
+            len = sendmsg(buffer)
+            buffer = buffer.byteslice(len...buffer.length)
+          rescue IO::WaitWritable, Errno::EINTR
+            yield_to_reactor(:write)
+          end
+        end
+        nil
+      end
+
+      # Detect support for socket#sendmsg_nonblock
+      Socket.new(Socket::AF_INET, Socket::SOCK_STREAM).tap do |socket|
+        begin
+          socket.sendmsg_nonblock('probe')
+        rescue Errno::EPIPE, Errno::ENOTCONN
+          def sendmsg(msg)
+            @io.sendmsg_nonblock(msg)
+          end
+        rescue NotImplementedError
+          def sendmsg(msg)
+            @io.write_nonblock(msg)
+          end
+        end
+      end
+
+      def yield_to_reactor(mode = :read)
+        if Fiber.yield(mode) == :close
+          throw :closed, 'Closed by reactor'
+        end
+      end
+
+      def pack(obj)
+        s = Marshal.dump(obj)
+        [s.bytesize].pack('N') << s
+      end
+
+      def yield_each(enumerator)
+        received_break = false
+        loop do
+          case msg = read_msg
+          when %w{NEXT}
+            # This will raise a StopIteration at the end of the enumeration,
+            # which will exit the loop.
+            write(enumerator.next)
+          when %w{BREAK}
+            # This is received when the client wants to stop the enumeration.
+            received_break = true
+            break
+          else
+            # Otherwise, the client is attempting to call another method within
+            # an `each` block.
+            write_dispatch(msg)
+          end
+        end
+      ensure
+        # This tells the client to stop enumerating
+        write(StopIteration.new("Server initiated stop")) unless received_break
+      end
+    end
+
     # @param [Hash] options
     # @option options [Integer] :port (9000) TCP port
     # @option options [String] :socket Alternative Unix socket file name
+    # @option options [Integer] :timeout (1) Number of seconds to timeout on IO.select
+    # @option options [Integer] :max_size (0x100000) Maximum number of bytes
+    #   allowed to be sent by clients in requests
     def initialize(store, options = {})
+      options = configure(**options)
       @store = store
-      @server = start(options)
+      @server = start(**options)
       @ios = [@server]
-      @clients = {}
+      @reads = @ios.dup
+      @writes = []
+      @connections = {}
       @running = false
     end
 
@@ -26,111 +183,116 @@ module Moneta
     #
     # @note This method blocks!
     def run
-      raise 'Already running' if @running
+      raise 'Already running' if running?
       @stop = false
       @running = true
       begin
-        until @stop
-          mainloop
-        end
+        mainloop until @stop
       ensure
-        File.unlink(@socket) if @socket
+        @running = false
+        @server.close unless @server.closed?
+        @ios
+          .reject { |io| io == @server }
+          .each { |io| close_connection(io) }
+        File.unlink(config.socket) if config.socket rescue nil
       end
     end
 
     # Stop the server
     def stop
-      raise 'Not running' unless @running
+      raise 'Not running' unless running?
       @stop = true
       @server.close
-      @server = nil
+      nil
     end
 
     private
 
-    TIMEOUT = 1
-    MAXSIZE = 0x100000
-
     def mainloop
-      if ios = IO.select(@ios, nil, @ios, TIMEOUT)
-        ios[2].each do |io|
-          io.close
-          delete_client(io)
-        end
-        ios[0].each do |io|
-          if io == @server
-            if client = @server.accept
-              @ios << client
-              @clients[client] = ''
-            end
-          elsif io.closed? || io.eof?
-            delete_client(io)
-          else
-            handle(io, @clients[io] << io.readpartial(0xFFFF))
-          end
+      if ready = IO.select(@reads, @writes, @ios, config.timeout)
+        reads, writes, errors = ready
+        errors.each { |io| close_connection(io) }
+
+        @reads -= reads
+        reads.each do |io|
+          io == @server ? accept_connection : resume(io)
         end
+
+        @writes -= writes
+        writes.each { |io| resume(io) }
       end
-    rescue SignalException => ex
-      warn "Moneta::Server - #{ex.message}"
-      raise if ex.signo == 15 || ex.signo == 2 # SIGTERM or SIGINT
-    rescue Exception => ex
+    rescue SignalException => signal
+      warn "Moneta::Server - received #{signal}"
+      case signal.signo
+      when Signal.list['INT'], Signal.list['TERM']
+        @stop = true # graceful exit
+      end
+    rescue IOError => ex
+      # We get a lot of these "closed stream" errors, which we ignore
+      raise unless ex.message =~ /closed stream/
+    rescue Errno::EBADF => ex
       warn "Moneta::Server - #{ex.message}"
     end
 
-    def delete_client(io)
+    def accept_connection
+      io = @server.accept
+      @connections[io] = Connection.new(io, @store, config.max_size)
+      @ios << io
+      resume(io)
+    ensure
+      @reads << @server
+    end
+
+    def delete_connection(io)
       @ios.delete(io)
-      @clients.delete(io)
+      @reads.delete(io)
+      @writes.delete(io)
     end
 
-    def pack(o)
-      s = Marshal.dump(o)
-      [s.bytesize].pack('N') << s
+    def close_connection(io)
+      delete_connection(io)
+      @connections.delete(io).resume(:close)
     end
 
-    def handle(io, buffer)
-      buffer = @clients[io]
-      return if buffer.bytesize < 8 # At least 4 bytes for the marshalled array
-      size = buffer[0,4].unpack('N').first
-      if size > MAXSIZE
-        delete_client(io)
-        return
-      end
-      return if buffer.bytesize < 4 + size
-      buffer.slice!(0, 4)
-      method, *args = Marshal.load(buffer.slice!(0, size))
-      case method
-      when :key?, :load, :delete, :increment, :create, :features
-        io.write(pack @store.send(method, *args))
-      when :store, :clear
-        @store.send(method, *args)
-        io.write(@nil ||= pack(nil))
-      else
-        raise 'Invalid method call'
+    def resume(io)
+      case result = @connections[io].resume
+      when :closed # graceful exit
+        delete_connection(io)
+      when Exception # messy exit
+        delete_connection(io)
+        raise result
+      when :read
+        @reads << io
+      when :write
+        @writes << io
       end
-    rescue IOError => ex
-      warn "Moneta::Server - #{ex.message}" unless ex.message =~ /closed/
-      delete_client(io)
-    rescue Exception => ex
-      warn "Moneta::Server - #{ex.message}"
-      io.write(pack Exception.new(ex.message))
     end
 
-    def start(options)
-      if @socket = options[:socket]
+    def start(host: '127.0.0.1', port: 9000, socket: nil)
+      if socket
         begin
-          UNIXServer.open(@socket)
+          UNIXServer.open(socket)
         rescue Errno::EADDRINUSE
-          if client = (UNIXSocket.open(@socket) rescue nil)
+          if client = (UNIXSocket.open(socket) rescue nil)
             client.close
             raise
           end
-          File.unlink(@socket)
+          File.unlink(socket)
           tries ||= 0
           (tries += 1) < 3 ? retry : raise
         end
       else
-        TCPServer.open(options[:host] || '127.0.0.1', options[:port] || 9000)
+        TCPServer.open(host, port)
       end
     end
+
+    def stats
+      {
+        connections: @connections.length,
+        reading: @reads.length,
+        writing: @writes.length,
+        total: @ios.length
+      }
+    end
   end
 end

data/lib/moneta/shared.rb

@@ -18,11 +18,12 @@ module Moneta
     def initialize(options = {}, &block)
       @options = options
       @builder = Builder.new(&block)
+      @connect_lock = ::Mutex.new
     end
 
     # (see Proxy#close)
     def close
-      if @server
+      if server?
         @server.stop
         @thread.join
         @server = @thread = nil
@@ -33,35 +34,56 @@ module Moneta
       end
     end
 
+    # Returns true if this wrapper is running as the server
+    #
+    # @return [Boolean] wrapper is a server
+    def server?
+      @server != nil
+    end
+
     protected
 
     def wrap(*args)
       connect
       yield
+    rescue Errno::ECONNRESET, Errno::EPIPE, IOError, SystemCallError
+      @connect_lock.synchronize { close unless server? }
+      tries ||= 0
+      (tries += 1) < 3 ? retry : raise
     end
 
     def connect
-      @adapter ||= Adapters::Client.new(@options)
-    rescue Errno::ECONNREFUSED, Errno::ENOENT => ex
+      return if @adapter
+      @connect_lock.synchronize do
+        @adapter ||= Adapters::Client.new(@options)
+      end
+    rescue Errno::ECONNREFUSED, Errno::ENOENT, IOError => ex
       start_server
       tries ||= 0
       warn "Moneta::Shared - Failed to connect: #{ex.message}" if tries > 0
-      (tries += 1) < 3 ? retry : raise
+      (tries += 1) < 10 ? retry : raise
     end
 
     # TODO: Implement this using forking (MRI) and threading (JRuby)
     # to get maximal performance
     def start_server
-      @adapter = Lock.new(@builder.build.last)
-      @server = Server.new(@adapter, @options)
-      @thread = Thread.new { @server.run }
-      sleep 0.1 until @server.running?
-    rescue Exception => ex
-      @adapter.close if @adapter
-      @adapter = nil
-      @server = nil
-      @thread = nil
-      warn "Moneta::Shared - Failed to start server: #{ex.message}"
+      @connect_lock.synchronize do
+        return if server?
+        begin
+          raise "Adapter already set" if @adapter
+          @adapter = Lock.new(@builder.build.last)
+          raise "Server already set" if server?
+          @server = Server.new(@adapter, @options)
+          @thread = Thread.new { @server.run }
+          sleep 0.1 until @server.running?
+        rescue => ex
+          @adapter.close if @adapter
+          @adapter = nil
+          @server = nil
+          @thread = nil
+          warn "Moneta::Shared - Failed to start server: #{ex.message}"
+        end
+      end
     end
   end
 end

data/lib/moneta/stack.rb

@@ -42,7 +42,7 @@ module Moneta
 
     # (see Proxy#key?)
     def key?(key, options = {})
-      @stack.any? {|s| s.key?(key, options) }
+      @stack.any? { |s| s.key?(key, options) }
     end
 
     # (see Proxy#load)
@@ -56,21 +56,21 @@ module Moneta
 
     # (see Proxy#store)
     def store(key, value, options = {})
-      @stack.each {|s| s.store(key, value, options) }
+      @stack.each { |s| s.store(key, value, options) }
       value
     end
 
     # (see Proxy#increment)
     def increment(key, amount = 1, options = {})
       last = nil
-      @stack.each {|s| last = s.increment(key, amount, options) }
+      @stack.each { |s| last = s.increment(key, amount, options) }
       last
     end
 
     # (see Proxy#create)
     def create(key, value, options = {})
       last = false
-      @stack.each {|s| last = s.create(key, value, options) }
+      @stack.each { |s| last = s.create(key, value, options) }
       last
     end
 
@@ -84,13 +84,13 @@ module Moneta
 
     # (see Proxy#clear)
     def clear(options = {})
-      @stack.each {|s| s.clear(options) }
+      @stack.each { |s| s.clear(options) }
       self
     end
 
     # (see Proxy#close)
     def close
-      @stack.each {|s| s.close }
+      @stack.each { |s| s.close }
       nil
     end
 
@@ -99,7 +99,7 @@ module Moneta
       @features ||=
         begin
           features = @stack.map(&:features)
-          features.inject(features.first, &:&).freeze
+          (features.inject(features.first, &:&) - [:each_key]).freeze
         end
     end
   end

data/lib/moneta/synchronize.rb

@@ -1,6 +1,6 @@
 module Moneta
   # Base class for {Mutex} and {Semaphore}
-  # @api private
+  # @abstract
   class SynchronizePrimitive
     # Synchronize block
     #
@@ -16,17 +16,15 @@ module Moneta
 
     # Try to enter critical section (nonblocking)
     #
-    # @api public
     # @return [Boolean] true if the lock was acquired
     def try_enter
       raise 'Already locked' if @locked
       enter_primitive ? @locked = true : false
     end
-    alias_method :try_lock, :try_enter
+    alias try_lock try_enter
 
     # Enter critical section (blocking)
     #
-    # @api public
     # @param [Number] timeout Maximum time to wait
     # @param [Number] wait Sleep time between tries to acquire lock
     # @return [Boolean] true if the lock was aquired
@@ -38,22 +36,18 @@ module Moneta
       end
       false
     end
-    alias_method :lock, :enter
+    alias lock enter
 
     # Leave critical section
-    #
-    # @api public
     def leave
       raise 'Not locked' unless @locked
       leave_primitive
       @locked = false
       nil
     end
-    alias_method :unlock, :leave
+    alias unlock leave
 
     # Is the lock acquired?
-    #
-    # @api public
     def locked?
       @locked
     end
@@ -101,7 +95,7 @@ module Moneta
   class Semaphore < SynchronizePrimitive
     # @param [Moneta store] store The store we want to lock
     # @param [Object] counter Key of the counter entry
-    # @param [Fixnum] max Maximum number of threads which are allowed to enter the critical section
+    # @param [Integer] max Maximum number of threads which are allowed to enter the critical section
     def initialize(store, counter, max = 1)
       raise 'Store must support feature :increment' unless store.supports?(:increment)
       @store, @counter, @max = store, counter, max