moneta 1.2.0 → 1.4.2

data/lib/moneta/hash_adapter.rb

@@ -0,0 +1,68 @@
+module Moneta
+  # @api private
+  module HashAdapter
+    attr_reader :backend
+
+    # (see Proxy#key?)
+    def key?(key, options = {})
+      @backend.has_key?(key)
+    end
+
+    # (see Proxy#load)
+    def load(key, options = {})
+      @backend[key]
+    end
+
+    # (see Proxy#store)
+    def store(key, value, options = {})
+      @backend[key] = value
+    end
+
+    # (see Proxy#delete)
+    def delete(key, options = {})
+      @backend.delete(key)
+    end
+
+    # (see Proxy#clear)
+    def clear(options = {})
+      @backend.clear
+      self
+    end
+
+    # (see Defaults#values_at)
+    def values_at(*keys, **options)
+      return super unless @backend.respond_to? :values_at
+      @backend.values_at(*keys)
+    end
+
+    # (see Defaults#fetch_values)
+    def fetch_values(*keys, **options, &defaults)
+      return super unless @backend.respond_to? :fetch_values
+      defaults ||= {} # prevents KeyError
+      @backend.fetch_values(*keys, &defaults)
+    end
+
+    # (see Defaults#slice)
+    def slice(*keys, **options)
+      return super unless @backend.respond_to? :slice
+      @backend.slice(*keys)
+    end
+
+    # (see Defaults#merge!)
+    def merge!(pairs, options = {}, &block)
+      return super unless method = [:merge!, :update].find do |method|
+        @backend.respond_to? method
+      end
+
+      hash = Hash === pairs ? pairs : Hash[pairs.to_a]
+      case method
+      when :merge!
+        @backend.merge!(hash, &block)
+      when :update
+        @backend.update(hash, &block)
+      end
+
+      self
+    end
+  end
+end

data/lib/moneta/increment_support.rb

@@ -0,0 +1,16 @@
+module Moneta
+  # @api private
+  module IncrementSupport
+    # (see Defaults#increment)
+    def increment(key, amount = 1, options = {})
+      existing = load(key, options)
+      value = (existing == nil ? 0 : Integer(existing)) + amount
+      store(key, value.to_s, options)
+      value
+    end
+
+    def self.included(base)
+      base.supports(:increment) if base.respond_to?(:supports)
+    end
+  end
+end

data/lib/moneta/lock.rb

@@ -15,6 +15,7 @@ module Moneta
     protected
 
     def wrap(name, *args, &block)
+      self.locks ||= Set.new
       if locked?
         yield
       else
@@ -22,8 +23,12 @@ module Moneta
       end
     end
 
+    def locks=(locks)
+      Thread.current.thread_variable_set('Moneta::Lock', locks)
+    end
+
     def locks
-      Thread.current['Moneta::Lock'] ||= Set.new
+      Thread.current.thread_variable_get('Moneta::Lock')
     end
 
     def lock!(&block)

data/lib/moneta/nil_values.rb

@@ -0,0 +1,35 @@
+module Moneta
+  # This contains overrides of methods in Defaults where additional nil
+  # checks are required, because nil values are possible in the store.
+  # @api private
+  module NilValues
+    def fetch_values(*keys, **options)
+      values = values_at(*keys, **options)
+      return values unless block_given?
+      keys.zip(values).map do |key, value|
+        if value == nil && !key?(key)
+          yield key
+        else
+          value
+        end
+      end
+    end
+
+    def slice(*keys, **options)
+      keys.zip(values_at(*keys, **options)).reject do |key, value|
+        value == nil && !key?(key)
+      end
+    end
+
+    def merge!(pairs, options = {})
+      pairs.each do |key, value|
+        if block_given? && key?(key, options)
+          existing = load(key, options)
+          value = yield(key, existing, value)
+        end
+        store(key, value, options)
+      end
+      self
+    end
+  end
+end

data/lib/moneta/option_support.rb

@@ -0,0 +1,51 @@
+module Moneta
+  # @api private
+  module OptionSupport
+    # Return Moneta store with default options or additional proxies
+    #
+    # @param [Hash] options Options to merge
+    # @return [Moneta store]
+    #
+    # @api public
+    def with(options = nil, &block)
+      adapter = self
+      if block
+        builder = Builder.new(&block)
+        builder.adapter(adapter)
+        adapter = builder.build.last
+      end
+      options ? OptionMerger.new(adapter, options) : adapter
+    end
+
+    # Return Moneta store with default option raw: true
+    #
+    # @return [OptionMerger]
+    # @api public
+    def raw
+      @raw ||=
+        begin
+          store = with(raw: true, only: [:load, :store, :create, :delete])
+          store.instance_variable_set(:@raw, store)
+          store
+        end
+    end
+
+    # Return Moneta store with default prefix option
+    #
+    # @param [String] prefix Key prefix
+    # @return [OptionMerger]
+    # @api public
+    def prefix(prefix)
+      with(prefix: prefix, except: :clear)
+    end
+
+    # Return Moneta store with default expiration time
+    #
+    # @param [Integer] expires Default expiration time
+    # @return [OptionMerger]
+    # @api public
+    def expires(expires)
+      with(expires: expires, only: [:store, :create, :increment])
+    end
+  end
+end

data/lib/moneta/pool.rb

@@ -87,6 +87,7 @@ module Moneta
         @waiting_since = [] if @timeout
         @last_checkout = nil
         @stopping = false
+        @idle_time = nil
 
         # Launch the manager thread
         @thread = run
@@ -126,10 +127,16 @@ module Moneta
             populate_stores
 
             until @stopping && @stores.empty?
+              loop_start = Time.now
+
               # Block until a message arrives, or until we time out for some reason
-              if request = pop
-                handle_request(request)
-              end
+              request = pop
+
+              # Record how long we were idle, for stats purposes
+              @idle_time = Time.now - loop_start
+
+              # If a message arrived, handle it
+              handle_request(request) if request
 
               # Handle any stale checkout requests
               handle_timed_out_requests
@@ -186,7 +193,7 @@ module Moneta
       # @return [Integer, nil]
       def timeout
         # Time to wait before there will be stores that should be closed
-        ttl = if @ttl && @last_checkout && !@available.empty?
+        ttl = if @ttl && @last_checkout && stores_available? && stores_unneeded?
                 [@ttl - (Time.now - @last_checkout), 0].max
               end
 
@@ -199,6 +206,18 @@ module Moneta
         [ttl, timeout].compact.min
       end
 
+      def stores_available?
+        !@available.empty?
+      end
+
+      def stores_unneeded?
+        @stores.length > @min
+      end
+
+      def stores_maxed?
+        @max != nil && @stores.length == @max
+      end
+
       def pop
         @mutex.synchronize do
           @resource.wait(@mutex, timeout) if @inbox.empty?
@@ -218,7 +237,7 @@ module Moneta
           reply.resolve(ShutdownError.new("Shutting down"))
         elsif !@available.empty?
           reply.resolve(@available.pop)
-        elsif !@max || @stores.length < @max
+        elsif !stores_maxed?
           begin
             reply.resolve(add_store)
           rescue => e
@@ -258,7 +277,8 @@ module Moneta
                       waiting: @waiting.length,
                       longest_wait: @timeout && !@waiting_since.empty? ? @waiting_since.first.dup : nil,
                       stopping: @stopping,
-                      last_checkout: @last_checkout && @last_checkout.dup)
+                      last_checkout: @last_checkout && @last_checkout.dup,
+                      idle_time: @idle_time.dup)
       end
 
       def handle_request(request)
@@ -298,6 +318,18 @@ module Moneta
     # by the manager after the ttl
     def close; end
 
+    def each_key(&block)
+      wrap(:each_key) do
+        raise NotImplementedError, "each_key is not supported on this proxy" \
+          unless supports? :each_key
+
+        return enum_for(:each_key) { adapter ? adapter.each_key.size : check_out! { adapter.each_key.size } } unless block_given?
+
+        adapter.each_key(&block)
+        self
+      end
+    end
+
     # Tells the manager to close all stores.  It will not be possible to use
     # the store after this.
     def stop

data/lib/moneta/proxy.rb

@@ -22,7 +22,7 @@ module Moneta
       raise NotImplementedError, "each_key is not supported on this proxy" \
         unless supports? :each_key
 
-      return enum_for(:each_key) unless block_given?
+      return enum_for(:each_key) { adapter.each_key.size } unless block_given?
       adapter.each_key(&block)
       self
     end

data/lib/moneta/server.rb

@@ -4,6 +4,154 @@ module Moneta
   # Moneta server to be used together with Moneta::Adapters::Client
   # @api public
   class Server
+    TIMEOUT = 1
+    MAXSIZE = 0x100000
+
+    # @api private
+    class Connection
+      def initialize(io, store)
+        @io = io
+        @store = store
+        @fiber = Fiber.new { run }
+      end
+
+      def resume(result = nil)
+        @fiber.resume result
+      end
+
+      private
+
+      # The return value of this function will be sent to the reactor.
+      #
+      # @return [:closed,Exception]
+      def run
+        catch :closed do
+          loop { write_dispatch(read_msg) }
+        end
+        :closed
+      rescue => ex
+        ex
+      ensure
+        @io.close unless @io.closed?
+      end
+
+      def dispatch(method, args)
+        case method
+        when :key?, :load, :delete, :increment, :create, :features
+          @store.public_send(method, *args)
+        when :store, :clear
+          @store.public_send(method, *args)
+          nil
+        when :each_key
+          yield_each(@store.each_key)
+          nil
+        end
+      rescue => ex
+        ex
+      end
+
+      def write_dispatch(msg)
+        method, *args = msg
+        result = dispatch(method, args)
+        write(result)
+      end
+
+      def read_msg
+        size = read(4).unpack('N').first
+        throw :closed, 'Message too big' if size > MAXSIZE
+        Marshal.load(read(size))
+      end
+
+      def read(len)
+        buffer = ''
+        loop do
+          begin
+            case received = @io.recv_nonblock(len)
+            when '', nil
+              throw :closed, 'Closed during read'
+            else
+              buffer << received
+              len -= received.bytesize
+            end
+          rescue IO::WaitReadable, IO::WaitWritable
+            yield_to_reactor(:read)
+          rescue Errno::ECONNRESET
+            throw :closed, 'Closed during read'
+          rescue IOError => ex
+            if ex.message =~ /closed stream/
+              throw :closed, 'Closed during read'
+            else
+              raise
+            end
+          end
+          break if len == 0
+        end
+        buffer
+      end
+
+      def write(obj)
+        buffer = pack(obj)
+        until buffer.empty?
+          begin
+            len = sendmsg(buffer)
+            buffer = buffer.byteslice(len...buffer.length)
+          rescue IO::WaitWritable, Errno::EINTR
+            yield_to_reactor(:write)
+          end
+        end
+        nil
+      end
+
+      # Detect support for socket#sendmsg_nonblock
+      Socket.new(Socket::AF_INET, Socket::SOCK_STREAM).tap do |socket|
+        begin
+          socket.sendmsg_nonblock('probe')
+        rescue Errno::EPIPE, Errno::ENOTCONN
+          def sendmsg(msg)
+            @io.sendmsg_nonblock(msg)
+          end
+        rescue NotImplementedError
+          def sendmsg(msg)
+            @io.write_nonblock(msg)
+          end
+        end
+      end
+
+      def yield_to_reactor(mode = :read)
+        if Fiber.yield(mode) == :close
+          throw :closed, 'Closed by reactor'
+        end
+      end
+
+      def pack(obj)
+        s = Marshal.dump(obj)
+        [s.bytesize].pack('N') << s
+      end
+
+      def yield_each(enumerator)
+        received_break = false
+        loop do
+          case msg = read_msg
+          when %w{NEXT}
+            # This will raise a StopIteration at the end of the enumeration,
+            # which will exit the loop.
+            write(enumerator.next)
+          when %w{BREAK}
+            # This is received when the client wants to stop the enumeration.
+            received_break = true
+            break
+          else
+            # Otherwise, the client is attempting to call another method within
+            # an `each` block.
+            write_dispatch(msg)
+          end
+        end
+      ensure
+        # This tells the client to stop enumerating
+        write(StopIteration.new("Server initiated stop")) unless received_break
+      end
+    end
+
     # @param [Hash] options
     # @option options [Integer] :port (9000) TCP port
     # @option options [String] :socket Alternative Unix socket file name
@@ -11,7 +159,9 @@ module Moneta
       @store = store
       @server = start(options)
       @ios = [@server]
-      @clients = {}
+      @reads = @ios.dup
+      @writes = []
+      @connections = {}
       @running = false
     end
 
@@ -26,94 +176,89 @@ module Moneta
     #
     # @note This method blocks!
     def run
-      raise 'Already running' if @running
+      raise 'Already running' if running?
       @stop = false
       @running = true
       begin
         mainloop until @stop
       ensure
-        File.unlink(@socket) if @socket
-        @ios.each { |io| io.close rescue nil }
+        @running = false
+        @server.close unless @server.closed?
+        @ios
+          .reject { |io| io == @server }
+          .each { |io| close_connection(io) }
+        File.unlink(@socket) if @socket rescue nil
       end
     end
 
     # Stop the server
     def stop
-      raise 'Not running' unless @running
+      raise 'Not running' unless running?
       @stop = true
       @server.close
-      @server = nil
+      nil
     end
 
     private
 
-    TIMEOUT = 1
-    MAXSIZE = 0x100000
-
     def mainloop
-      if ios = IO.select(@ios, nil, @ios, TIMEOUT)
-        ios[2].each do |io|
-          io.close
-          delete_client(io)
-        end
-        ios[0].each do |io|
-          if io == @server
-            if client = @server.accept
-              @ios << client
-              @clients[client] = ''
-            end
-          elsif io.closed? || io.eof?
-            delete_client(io)
-          else
-            handle(io, @clients[io] << io.readpartial(0xFFFF))
-          end
+      if ready = IO.select(@reads, @writes, @ios, TIMEOUT)
+        reads, writes, errors = ready
+        errors.each { |io| close_connection(io) }
+
+        @reads -= reads
+        reads.each do |io|
+          io == @server ? accept_connection : resume(io)
         end
+
+        @writes -= writes
+        writes.each { |io| resume(io) }
       end
-    rescue SignalException => ex
-      warn "Moneta::Server - #{ex.message}"
-      raise if ex.signo == 15 || ex.signo == 2 # SIGTERM or SIGINT
-    rescue => ex
+    rescue SignalException => signal
+      warn "Moneta::Server - received #{signal}"
+      case signal.signo
+      when Signal.list['INT'], Signal.list['TERM']
+        @stop = true # graceful exit
+      end
+    rescue IOError => ex
+      # We get a lot of these "closed stream" errors, which we ignore
+      raise unless ex.message =~ /closed stream/
+    rescue Errno::EBADF => ex
       warn "Moneta::Server - #{ex.message}"
     end
 
-    def delete_client(io)
+    def accept_connection
+      io = @server.accept
+      @connections[io] = Connection.new(io, @store)
+      @ios << io
+      resume(io)
+    ensure
+      @reads << @server
+    end
+
+    def delete_connection(io)
       @ios.delete(io)
-      @clients.delete(io)
+      @reads.delete(io)
+      @writes.delete(io)
     end
 
-    def pack(obj)
-      s = Marshal.dump(obj)
-      [s.bytesize].pack('N') << s
+    def close_connection(io)
+      delete_connection(io)
+      @connections.delete(io).resume(:close)
     end
 
-    def handle(io, buffer)
-      return if buffer.bytesize < 8 # At least 4 bytes for the marshalled array
-      size = buffer[0, 4].unpack('N').first
-      if size > MAXSIZE
-        delete_client(io)
-        return
-      end
-      return if buffer.bytesize < 4 + size
-      buffer.slice!(0, 4)
-      method, *args = Marshal.load(buffer.slice!(0, size))
-      case method
-      when :key?, :load, :delete, :increment, :create
-        io.write(pack(@store.send(method, *args)))
-      when :features
-        # all features except each_key are supported
-        io.write(pack(@store.features - [:each_key]))
-      when :store, :clear
-        @store.send(method, *args)
-        io.write(@nil ||= pack(nil))
-      else
-        raise 'Invalid method call'
+    def resume(io)
+      case result = @connections[io].resume
+      when :closed # graceful exit
+        delete_connection(io)
+      when Exception # messy exit
+        delete_connection(io)
+        raise result
+      when :read
+        @reads << io
+      when :write
+        @writes << io
       end
-    rescue IOError => ex
-      warn "Moneta::Server - #{ex.message}" unless ex.message =~ /closed/
-      delete_client(io)
-    rescue => ex
-      warn "Moneta::Server - #{ex.message}"
-      io.write(pack(Exception.new(ex.message)))
     end
 
     def start(options)
@@ -133,5 +278,14 @@ module Moneta
         TCPServer.open(options[:host] || '127.0.0.1', options[:port] || 9000)
       end
     end
+
+    def stats
+      {
+        connections: @connections.length,
+        reading: @reads.length,
+        writing: @writes.length,
+        total: @ios.length
+      }
+    end
   end
 end