moneta 1.1.0 → 1.4.0

data/lib/moneta/expires.rb

@@ -21,14 +21,14 @@ module Moneta
     def key?(key, options = {})
       # Transformer might raise exception
       load_entry(key, options) != nil
-    rescue Exception
+    rescue
       super(key, Utils.without(options, :expires))
     end
 
     # (see Proxy#load)
     def load(key, options = {})
       return super if options.include?(:raw)
-      value, expires = load_entry(key, options)
+      value, = load_entry(key, options)
       value
     end
 
@@ -64,8 +64,8 @@ module Moneta
           entry = invalidate_entry(key, entry, new_expires) do |new_entry|
             updates[key] = new_entry
           end
-          next if entry.nil?
-          value, _ = entry
+          next if entry == nil
+          value, = entry
           value
         end
       end
@@ -90,12 +90,12 @@ module Moneta
           entry = invalidate_entry(key, entry, new_expires) do |new_entry|
             updates[key] = new_entry
           end
-          if entry.nil?
+          if entry == nil
             value = if block_given?
                       yield key
                     end
           else
-            value, _ = entry
+            value, = entry
           end
           value
         end
@@ -113,26 +113,26 @@ module Moneta
           entry = invalidate_entry(key, entry, new_expires) do |new_entry|
             updates[key] = new_entry
           end
-          next if entry.nil?
-          value, _ = entry
+          next if entry == nil
+          value, = entry
           [key, value]
         end.reject(&:nil?)
       end
     end
 
     # (see Defaults#merge!)
-    def merge!(pairs, options={})
+    def merge!(pairs, options = {})
       expires = expires_at(options)
       options = Utils.without(options, :expires)
 
       block = if block_given?
                 lambda do |key, old_entry, entry|
                   old_entry = invalidate_entry(key, old_entry)
-                  if old_entry.nil?
+                  if old_entry == nil
                     entry # behave as if no replace is happening
                   else
-                    old_value, _ = old_entry
-                    new_value, _ = entry
+                    old_value, = old_entry
+                    new_value, = entry
                     new_entry(yield(key, old_value, new_value), expires)
                   end
                 end

data/lib/moneta/expires_support.rb

@@ -0,0 +1,60 @@
+module Moneta
+  # This mixin handles the calculation of expiration times.
+  #
+  #
+  module ExpiresSupport
+    attr_accessor :default_expires
+
+    protected
+
+    # Calculates the time when something will expire.
+    #
+    # This method considers false and 0 as "no-expire" and every positive
+    # number as a time to live in seconds.
+    #
+    # @param [Hash] options Options hash
+    # @option options [0,false,nil,Numeric] :expires expires value given by user
+    # @param [0,false,nil,Numeric] default default expiration time
+    #
+    # @return [false] if it should not expire
+    # @return [Time] the time when something should expire
+    # @return [nil] if it is not known
+    def expires_at(options, default = @default_expires)
+      value = expires_value(options, default)
+      Numeric === value ? Time.now + value : value
+    end
+
+    # Calculates the number of seconds something should last.
+    #
+    # This method considers false and 0 as "no-expire" and every positive
+    # number as a time to live in seconds.
+    #
+    # @param [Hash] options Options hash
+    # @option options [0,false,nil,Numeric] :expires expires value given by user
+    # @param [0,false,nil,Numeric] default default expiration time
+    #
+    # @return [false] if it should not expire
+    # @return [Numeric] seconds until expiration
+    # @return [nil] if it is not known
+    def expires_value(options, default = @default_expires)
+      case value = options[:expires]
+      when 0, false
+        false
+      when nil
+        default ? default.to_r : nil
+      when Numeric
+        value = value.to_r
+        raise ArgumentError, ":expires must be a positive value, got #{value}" if value < 0
+        value
+      else
+        raise ArgumentError, ":expires must be Numeric or false, got #{value.inspect}"
+      end
+    end
+
+    class << self
+      def included(base)
+        base.supports(:expires) if base.respond_to?(:supports)
+      end
+    end
+  end
+end

data/lib/moneta/fallback.rb

@@ -0,0 +1,84 @@
+module Moneta
+  # Provides a fallback to a second store when an exception is raised
+  #
+  # @example Basic usage - catches any {IOError} and falls back to {Moneta::Adapters:Null}
+  #   Moneta.build do
+  #     use :Fallback
+  #     adapter :Client
+  #   end
+  #
+  # @example Specifying an exception to rescue
+  #   Moneta.build do
+  #     use :Fallback, rescue: Redis::CannotConnectError
+  #     adapter :Redis
+  #   end
+  #
+  # @example Specifying a different fallback
+  #   Moneta.build do
+  #     use :Fallback do
+  #       # This is a new builder context
+  #       adapter :Memory
+  #     end
+  #     adapter :File, dir: 'cache'
+  #   end
+  #
+  # @api public
+  class Fallback < Wrapper
+    # @param [Moneta store] adapter The underlying store
+    # @param [Hash] options
+    # @option options [Moneta store] :fallback (:Null store) The store to fall
+    #   back on
+    # @option options [Class|Array<Class>] :rescue ([IOError]) The list
+    #   of exceptions that should be rescued
+    # @yieldreturn [Moneta store] Moneta store built using the builder API
+    def initialize(adapter, options = {}, &block)
+      super
+
+      @fallback =
+        if block_given?
+          ::Moneta.build(&block)
+        elsif options.key?(:fallback)
+          options.delete(:fallback)
+        else
+          ::Moneta::Adapters::Null.new
+        end
+
+      @rescue =
+        case options[:rescue]
+        when nil
+          [::IOError]
+        when Array
+          options[:rescue]
+        else
+          [options[:rescue]]
+        end
+    end
+
+    protected
+
+    def wrap(name, *args, &block)
+      yield
+    rescue => e
+      raise unless @rescue.any? { |rescuable| rescuable === e }
+      fallback(name, *args, &block)
+    end
+
+    def fallback(name, *args, &block)
+      result =
+        case name
+        when :values_at, :fetch_values, :slice
+          keys, options = args
+          @fallback.public_send(name, *keys, **options, &block)
+        else
+          @fallback.public_send(name, *args, &block)
+        end
+
+      # Don't expose the fallback class to the caller
+      if result == @fallback
+        self
+      else
+        result
+      end
+    end
+  end
+end

data/lib/moneta/hash_adapter.rb

@@ -0,0 +1,68 @@
+module Moneta
+  # @api private
+  module HashAdapter
+    attr_reader :backend
+
+    # (see Proxy#key?)
+    def key?(key, options = {})
+      @backend.has_key?(key)
+    end
+
+    # (see Proxy#load)
+    def load(key, options = {})
+      @backend[key]
+    end
+
+    # (see Proxy#store)
+    def store(key, value, options = {})
+      @backend[key] = value
+    end
+
+    # (see Proxy#delete)
+    def delete(key, options = {})
+      @backend.delete(key)
+    end
+
+    # (see Proxy#clear)
+    def clear(options = {})
+      @backend.clear
+      self
+    end
+
+    # (see Defaults#values_at)
+    def values_at(*keys, **options)
+      return super unless @backend.respond_to? :values_at
+      @backend.values_at(*keys)
+    end
+
+    # (see Defaults#fetch_values)
+    def fetch_values(*keys, **options, &defaults)
+      return super unless @backend.respond_to? :fetch_values
+      defaults ||= {} # prevents KeyError
+      @backend.fetch_values(*keys, &defaults)
+    end
+
+    # (see Defaults#slice)
+    def slice(*keys, **options)
+      return super unless @backend.respond_to? :slice
+      @backend.slice(*keys)
+    end
+
+    # (see Defaults#merge!)
+    def merge!(pairs, options = {}, &block)
+      return super unless method = [:merge!, :update].find do |method|
+        @backend.respond_to? method
+      end
+
+      hash = Hash === pairs ? pairs : Hash[pairs.to_a]
+      case method
+      when :merge!
+        @backend.merge!(hash, &block)
+      when :update
+        @backend.update(hash, &block)
+      end
+
+      self
+    end
+  end
+end

data/lib/moneta/increment_support.rb

@@ -0,0 +1,16 @@
+module Moneta
+  # @api private
+  module IncrementSupport
+    # (see Defaults#increment)
+    def increment(key, amount = 1, options = {})
+      existing = load(key, options)
+      value = (existing == nil ? 0 : Integer(existing)) + amount
+      store(key, value.to_s, options)
+      value
+    end
+
+    def self.included(base)
+      base.supports(:increment) if base.respond_to?(:supports)
+    end
+  end
+end

data/lib/moneta/lock.rb

@@ -15,15 +15,20 @@ module Moneta
     protected
 
     def wrap(name, *args, &block)
+      self.locks ||= Set.new
       if locked?
-        block.call
+        yield
       else
         lock!(&block)
       end
     end
 
+    def locks=(locks)
+      Thread.current.thread_variable_set('Moneta::Lock', locks)
+    end
+
     def locks
-      Thread.current['Moneta::Lock'] ||= Set.new
+      Thread.current.thread_variable_get('Moneta::Lock')
     end
 
     def lock!(&block)

data/lib/moneta/logger.rb

@@ -29,7 +29,7 @@ module Moneta
       def format(entry)
         args = entry[:args]
         args.pop if Hash === args.last && args.last.empty?
-        args = args.map {|a| dump(a) }.join(', ')
+        args = args.map { |a| dump(a) }.join(', ')
         if entry[:error]
           "#{@prefix}#{entry[:method]}(#{args}) raised error: #{entry[:error].message}\n"
         else
@@ -66,7 +66,7 @@ module Moneta
       ret = yield
       @logger.log(method: method, args: args, return: (method == :clear ? 'self' : ret))
       ret
-    rescue Exception => error
+    rescue => error
       @logger.log(method: method, args: args, error: error)
       raise
     end

data/lib/moneta/nil_values.rb

@@ -0,0 +1,35 @@
+module Moneta
+  # This contains overrides of methods in Defaults where additional nil
+  # checks are required, because nil values are possible in the store.
+  # @api private
+  module NilValues
+    def fetch_values(*keys, **options)
+      values = values_at(*keys, **options)
+      return values unless block_given?
+      keys.zip(values).map do |key, value|
+        if value == nil && !key?(key)
+          yield key
+        else
+          value
+        end
+      end
+    end
+
+    def slice(*keys, **options)
+      keys.zip(values_at(*keys, **options)).reject do |key, value|
+        value == nil && !key?(key)
+      end
+    end
+
+    def merge!(pairs, options = {})
+      pairs.each do |key, value|
+        if block_given? && key?(key, options)
+          existing = load(key, options)
+          value = yield(key, existing, value)
+        end
+        store(key, value, options)
+      end
+      self
+    end
+  end
+end

data/lib/moneta/option_support.rb

@@ -0,0 +1,51 @@
+module Moneta
+  # @api private
+  module OptionSupport
+    # Return Moneta store with default options or additional proxies
+    #
+    # @param [Hash] options Options to merge
+    # @return [Moneta store]
+    #
+    # @api public
+    def with(options = nil, &block)
+      adapter = self
+      if block
+        builder = Builder.new(&block)
+        builder.adapter(adapter)
+        adapter = builder.build.last
+      end
+      options ? OptionMerger.new(adapter, options) : adapter
+    end
+
+    # Return Moneta store with default option raw: true
+    #
+    # @return [OptionMerger]
+    # @api public
+    def raw
+      @raw ||=
+        begin
+          store = with(raw: true, only: [:load, :store, :create, :delete])
+          store.instance_variable_set(:@raw, store)
+          store
+        end
+    end
+
+    # Return Moneta store with default prefix option
+    #
+    # @param [String] prefix Key prefix
+    # @return [OptionMerger]
+    # @api public
+    def prefix(prefix)
+      with(prefix: prefix, except: :clear)
+    end
+
+    # Return Moneta store with default expiration time
+    #
+    # @param [Integer] expires Default expiration time
+    # @return [OptionMerger]
+    # @api public
+    def expires(expires)
+      with(expires: expires, only: [:store, :create, :increment])
+    end
+  end
+end

data/lib/moneta/optionmerger.rb

@@ -40,4 +40,3 @@ module Moneta
     end
   end
 end
-

data/lib/moneta/pool.rb

@@ -1,69 +1,351 @@
-require 'thread'
+require 'set'
 
 module Moneta
-  # Creates a pool of stores.
-  # Each thread gets its own store.
+  # Creates a thread-safe pool.  Stores are in the pool are transparently
+  # checked in and out in order to perform operations.
+  #
+  # A `max` setting can be specified in order to limit the pool size.  If `max`
+  # stores are all checked out at once, the next check-out will block until one
+  # of the other stores are checked in.
+  #
+  # A `ttl` setting can be specified, giving the number of seconds to
+  # wait without any activity before shrinking the pool size back down to the
+  # min size.
+  #
+  # A `timeout` setting can be specified, giving the number of seconds to wait
+  # when checking out a store, before an error is raised.  When the pool has a
+  # `:max` size, a timeout is highly advisable.
   #
   # @example Add `Moneta::Pool` to proxy stack
   #   Moneta.build do
   #     use(:Pool) do
-  #       # Every thread gets its own instance
   #       adapter :MemcachedNative
   #     end
   #   end
   #
+  # @example Add `Moneta::Pool` that contains at least 2 stores, and closes any extras after 60 seconds of inactivity
+  #   Moneta.build do
+  #     use(:Pool, min: 2, ttl: 60) do
+  #       adapter :Sqlite, file: 'test.db'
+  #     end
+  #   end
+  #
+  # @example Add `Moneta::Pool` with a max of 10 stores, and a timeout of 5 seconds for checkout
+  #   Moneta.build do
+  #     use(:Pool, max: 10, timeout: 5) do
+  #       adapter :Sqlite, file: 'test.db'
+  #     end
+  #   end
+  #
   # @api public
   class Pool < Wrapper
+    # @api private
+    class ShutdownError < ::RuntimeError; end
+    class TimeoutError < ::RuntimeError; end
+
+    # @api private
+    class Reply
+      attr_reader :resource
+
+      def initialize(mutex)
+        @mutex = mutex
+        @resource = ::ConditionVariable.new
+        @value = nil
+      end
+
+      def resolve(value)
+        @mutex.synchronize do
+          raise "Already resolved" if @value
+          @value = value
+          @resource.signal
+        end
+        nil
+      end
+
+      def wait
+        @resource.wait(@mutex)
+        @value
+      end
+    end
+
+    # @api private
+    class PoolManager
+      def initialize(builder, min: 0, max: nil, ttl: nil, timeout: nil)
+        @builder = builder
+        @min = min
+        @max = max
+        @ttl = ttl
+        @timeout = timeout
+
+        @inbox = []
+        @mutex = ::Mutex.new
+        @resource = ::ConditionVariable.new
+
+        @stores = Set.new
+        @available = []
+        @waiting = []
+        @waiting_since = [] if @timeout
+        @last_checkout = nil
+        @stopping = false
+
+        # Launch the manager thread
+        @thread = run
+      end
+
+      def stats
+        push(:stats, reply: true)
+      end
+
+      def stop
+        push(:stop)
+        nil
+      ensure
+        @thread.value
+      end
+
+      def kill!
+        @thread.kill
+        nil
+      end
+
+      def check_out
+        reply = push(:check_out, reply: true)
+        raise reply if Exception === reply
+        reply
+      end
+
+      def check_in(store)
+        push(:check_in, store)
+      end
+
+      private
+
+      def run
+        Thread.new do
+          begin
+            populate_stores
+
+            until @stopping && @stores.empty?
+              # Block until a message arrives, or until we time out for some reason
+              if request = pop
+                handle_request(request)
+              end
+
+              # Handle any stale checkout requests
+              handle_timed_out_requests
+              # Drop any stores that are no longer needed
+              remove_unneeded_stores
+            end
+          rescue => e
+            reject_waiting(e.message)
+            raise
+          end
+        end
+      end
+
+      def populate_stores
+        return if @stopping
+        @available.push(add_store) while @stores.length < @min
+      end
+
+      # If the last checkout was more than timeout ago, drop any available stores
+      def remove_unneeded_stores
+        return unless @stopping || (@ttl && @last_checkout && Time.now - @last_checkout >= @ttl)
+        while (@stopping || @stores.length > @min) and store = @available.pop
+          store.close rescue nil
+          @stores.delete(store)
+        end
+      end
+
+      # If there are checkout requests that have been waiting too long,
+      # feed them timeout errors.
+      def handle_timed_out_requests
+        while @timeout && !@waiting.empty? && (Time.now - @waiting_since.first) >= @timeout
+          waiting_since = @waiting_since.shift
+          @waiting.shift.resolve(TimeoutError.new("Waited %<secs>f seconds" % { secs: Time.now - waiting_since }))
+        end
+      end
+
+      # This is called from outside the loop thread
+      def push(message, what = nil, reply: nil)
+        @mutex.synchronize do
+          raise ShutdownError, "Pool has been shutdown" if reply && !@thread.alive?
+          reply &&= Reply.new(@mutex)
+          @inbox.push([message, what, reply])
+          @resource.signal
+          reply.wait if reply
+        end
+      end
+
+      # This method calculates the number of seconds to wait for a signal on
+      # the condition variable, or `nil` if there is no need to time out.
+      #
+      # Calculated based on the `:ttl` and `:timeout` options used during
+      # construction.
+      #
+      # @return [Integer, nil]
+      def timeout
+        # Time to wait before there will be stores that should be closed
+        ttl = if @ttl && @last_checkout && !@available.empty?
+                [@ttl - (Time.now - @last_checkout), 0].max
+              end
+
+        # Time to wait
+        timeout = if @timeout && !@waiting_since.empty?
+                    longest_waiting = @waiting_since.first
+                    [@timeout - (Time.now - longest_waiting), 0].max
+                  end
+
+        [ttl, timeout].compact.min
+      end
+
+      def pop
+        @mutex.synchronize do
+          @resource.wait(@mutex, timeout) if @inbox.empty?
+          @inbox.shift
+        end
+      end
+
+      def add_store
+        store = @builder.build.last
+        @stores.add(store)
+        store
+      end
+
+      def handle_check_out(reply)
+        @last_checkout = Time.now
+        if @stopping
+          reply.resolve(ShutdownError.new("Shutting down"))
+        elsif !@available.empty?
+          reply.resolve(@available.pop)
+        elsif !@max || @stores.length < @max
+          begin
+            reply.resolve(add_store)
+          rescue => e
+            reply.resolve(e)
+          end
+        else
+          @waiting.push(reply)
+          @waiting_since.push(Time.now) if @timeout
+        end
+      end
+
+      def handle_stop
+        @stopping = true
+        # Reject anyone left waiting
+        reject_waiting "Shutting down"
+      end
+
+      def reject_waiting(reason)
+        while reply = @waiting.shift
+          reply.resolve(ShutdownError.new(reason))
+        end
+        @waiting_since = [] if @timeout
+      end
+
+      def handle_check_in(store)
+        if !@waiting.empty?
+          @waiting.shift.resolve(store)
+          @waiting_since.shift if @timeout
+        else
+          @available.push(store)
+        end
+      end
+
+      def handle_stats(reply)
+        reply.resolve(stores: @stores.length,
+                      available: @available.length,
+                      waiting: @waiting.length,
+                      longest_wait: @timeout && !@waiting_since.empty? ? @waiting_since.first.dup : nil,
+                      stopping: @stopping,
+                      last_checkout: @last_checkout && @last_checkout.dup)
+      end
+
+      def handle_request(request)
+        cmd, what, reply = request
+        case cmd
+        when :check_out
+          handle_check_out(reply)
+        when :check_in
+          # A checkin request
+          handle_check_in(what)
+        when :stats
+          handle_stats(reply)
+        when :stop
+          # Graceful exit
+          handle_stop
+        end
+      end
+    end
+
     # @param [Hash] options
-    # @option options [String] :mutex (::Mutex.new) Mutex object
+    # @option options [Integer] :min (0) The minimum pool size
+    # @option options [Integer] :max The maximum pool size.  If not specified,
+    #   there is no maximum.
+    # @option options [Numeric] :ttl The number of seconds to keep
+    #   stores above the minumum number around for without activity.  If
+    #   not specified, stores will never be removed.
+    # @option options [Numeric] :timeout The number of seconds to wait for a
+    #   store to become available.  If not specified, will wait forever.
+    # @yield A builder context for speciying how to construct stores
     def initialize(options = {}, &block)
       super(nil)
-      @mutex = options[:mutex] || ::Mutex.new
       @id = "Moneta::Pool(#{object_id})"
-      @builder = Builder.new(&block)
-      @pool, @all = [], []
+      @manager = PoolManager.new(Builder.new(&block), **options)
     end
 
-    def close
-      @mutex.synchronize do
-        raise '#close can only be called when no thread is using the pool' if @all.size != @pool.size
-        @all.each(&:close)
-        @all = @pool = nil
+    # Closing has no effect on the pool, as stores are closed in the background
+    # by the manager after the ttl
+    def close; end
+
+    def each_key(&block)
+      wrap(:each_key) do
+        raise NotImplementedError, "each_key is not supported on this proxy" \
+          unless supports? :each_key
+
+        return enum_for(:each_key) { adapter ? adapter.each_key.size : check_out! { adapter.each_key.size } } unless block_given?
+
+        adapter.each_key(&block)
+        self
       end
     end
 
+    # Tells the manager to close all stores.  It will not be possible to use
+    # the store after this.
+    def stop
+      @manager.stop
+      nil
+    end
+
+    def stats
+      @manager.stats
+    end
+
     protected
 
     def adapter
-      Thread.current[@id]
+      Thread.current.thread_variable_get(@id)
+    end
+
+    def adapter=(store)
+      Thread.current.thread_variable_set(@id, store)
     end
 
     def wrap(*args, &block)
-      if checked_out?
+      if adapter
         yield
       else
         check_out!(&block)
       end
     end
 
-    def checked_out?
-      Thread.current.key?(@id) && Thread.current[@id] != nil
-    end
-
     def check_out!
-      store = Thread.current[@id] = pop
+      store = @manager.check_out
+      self.adapter = store
       yield
     ensure
-      Thread.current[@id] = nil
-      @mutex.synchronize { @pool << store }
-    end
-
-    def pop
-      unless store = @mutex.synchronize { @pool.pop }
-        store = @builder.build.last
-        @mutex.synchronize { @all << store }
-      end
-      store
+      self.adapter = nil
+      @manager.check_in store if store
     end
   end
 end