moneta 1.0.0 → 1.5.1

data/lib/moneta/adapters/mongo.rb

@@ -1,11 +1,270 @@
+require 'mongo'
+
 module Moneta
   module Adapters
-    begin
-      require 'moneta/adapters/mongo/official'
-      Mongo = MongoOfficial
-    rescue LoadError
-      require 'moneta/adapters/mongo/moped'
-      Mongo = MongoMoped
+    # MongoDB backend
+    #
+    # Supports expiration, documents will be automatically removed starting
+    # with mongodb >= 2.2 (see {http://docs.mongodb.org/manual/tutorial/expire-data/}).
+    #
+    # You can store hashes directly using this adapter.
+    #
+    # @example Store hashes
+    #     db = Moneta::Adapters::MongoOfficial.new
+    #     db['key'] = {a: 1, b: 2}
+    #
+    # @api public
+    class Mongo < Adapter
+      include ExpiresSupport
+
+      supports :each_key, :create, :increment
+
+      config :collection, default: 'moneta'
+
+      config :db
+      config :database, default: 'moneta' do |database:, db:, **|
+        if db
+          warn('Moneta::Adapters::Mongo - the :db option is deprecated and will be removed in a future version. Use :database instead')
+          db
+        else
+          database
+        end
+      end
+
+      config :expires_field, default: 'expiresAt'
+      config :value_field, default: 'value'
+      config :type_field, default: 'type'
+
+      backend do |host: '127.0.0.1', port: 27017, **options|
+        options[:logger] ||= ::Logger.new(STDERR).tap do |logger|
+          logger.level = ::Logger::ERROR
+        end
+        ::Mongo::Client.new(["#{host}:#{port}"], options)
+      end
+
+      # @param [Hash] options
+      # @option options [String] :collection ('moneta') MongoDB collection name
+      # @option options [String] :host ('127.0.0.1') MongoDB server host
+      # @option options [String] :user Username used to authenticate
+      # @option options [String] :password Password used to authenticate
+      # @option options [Integer] :port (MongoDB default port) MongoDB server port
+      # @option options [String] :database ('moneta') MongoDB database
+      # @option options [Integer] :expires Default expiration time
+      # @option options [String] :expires_field ('expiresAt') Document field to store expiration time
+      # @option options [String] :value_field ('value') Document field to store value
+      # @option options [String] :type_field ('type') Document field to store value type
+      # @option options [::Mongo::Client] :backend Use existing backend instance
+      # @option options Other options passed to `Mongo::MongoClient#new`
+      def initialize(options = {})
+        super
+
+        @database = backend.use(config.database)
+        @collection = @database[config.collection]
+
+        if @database.command(buildinfo: 1).documents.first['version'] >= '2.2'
+          @collection.indexes.create_one({ config.expires_field => 1 }, expire_after: 0)
+        else
+          warn 'Moneta::Adapters::Mongo - You are using MongoDB version < 2.2, expired documents will not be deleted'
+        end
+      end
+
+      # (see Proxy#load)
+      def load(key, options = {})
+        view = @collection.find(:$and => [
+                                  { _id: to_binary(key) },
+                                  not_expired
+                                ])
+
+        doc = view.limit(1).first
+
+        if doc
+          update_expiry(options, nil) do |expires|
+            view.update_one(:$set => { config.expires_field => expires })
+          end
+
+          doc_to_value(doc)
+        end
+      end
+
+      # (see Proxy#store)
+      def store(key, value, options = {})
+        key = to_binary(key)
+        @collection.replace_one({ _id: key },
+                                value_to_doc(key, value, options),
+                                upsert: true)
+        value
+      end
+
+      # (see Proxy#each_key)
+      def each_key
+        return enum_for(:each_key) unless block_given?
+        @collection.find.each { |doc| yield from_binary(doc[:_id]) }
+        self
+      end
+
+      # (see Proxy#delete)
+      def delete(key, options = {})
+        key = to_binary(key)
+        if doc = @collection.find(_id: key).find_one_and_delete and
+            !doc[config.expires_field] || doc[config.expires_field] >= Time.now
+          doc_to_value(doc)
+        end
+      end
+
+      # (see Proxy#increment)
+      def increment(key, amount = 1, options = {})
+        @collection.find_one_and_update({ :$and => [{ _id: to_binary(key) }, not_expired] },
+                                        { :$inc => { config.value_field => amount } },
+                                        return_document: :after,
+                                        upsert: true)[config.value_field]
+      rescue ::Mongo::Error::OperationFailure
+        tries ||= 0
+        (tries += 1) < 3 ? retry : raise
+      end
+
+      # (see Proxy#create)
+      def create(key, value, options = {})
+        key = to_binary(key)
+        @collection.insert_one(value_to_doc(key, value, options))
+        true
+      rescue ::Mongo::Error::OperationFailure => error
+        raise unless error.code == 11000 # duplicate key error
+        false
+      end
+
+      # (see Proxy#clear)
+      def clear(options = {})
+        @collection.delete_many
+        self
+      end
+
+      # (see Proxy#close)
+      def close
+        @database.close
+        nil
+      end
+
+      # (see Proxy#slice)
+      def slice(*keys, **options)
+        view = @collection.find(:$and => [
+                                  { _id: { :$in => keys.map(&method(:to_binary)) } },
+                                  not_expired
+                                ])
+        pairs = view.map { |doc| [from_binary(doc[:_id]), doc_to_value(doc)] }
+
+        update_expiry(options, nil) do |expires|
+          view.update_many(:$set => { config.expires_field => expires })
+        end
+
+        pairs
+      end
+
+      # (see Proxy#merge!)
+      def merge!(pairs, options = {})
+        existing = Hash[slice(*pairs.map { |key, _| key })]
+        update_pairs, insert_pairs = pairs.partition { |key, _| existing.key?(key) }
+
+        @collection.insert_many(insert_pairs.map do |key, value|
+          value_to_doc(to_binary(key), value, options)
+        end)
+
+        update_pairs.each do |key, value|
+          value = yield(key, existing[key], value) if block_given?
+          binary = to_binary(key)
+          @collection.replace_one({ _id: binary }, value_to_doc(binary, value, options))
+        end
+
+        self
+      end
+
+      # (see Proxy#fetch_values)
+      def fetch_values(*keys, **options)
+        return values_at(*keys, **options) unless block_given?
+        hash = Hash[slice(*keys, **options)]
+        keys.map do |key|
+          if hash.key?(key)
+            hash[key]
+          else
+            yield key
+          end
+        end
+      end
+
+      # (see Proxy#values_at)
+      def values_at(*keys, **options)
+        hash = Hash[slice(*keys, **options)]
+        keys.map { |key| hash[key] }
+      end
+
+      private
+
+      def doc_to_value(doc)
+        case doc[config.type_field]
+        when 'Hash'
+          doc = doc.dup
+          doc.delete('_id')
+          doc.delete(config.type_field)
+          doc.delete(config.expires_field)
+          doc
+        when 'Number'
+          doc[config.value_field]
+        else
+          # In ruby_bson version 2 (and probably up), #to_s no longer returns the binary data
+          from_binary(doc[config.value_field])
+        end
+      end
+
+      def value_to_doc(key, value, options)
+        case value
+        when Hash
+          value.merge('_id' => key,
+                      config.type_field => 'Hash',
+                      # expires_field must be a Time object (BSON date datatype)
+                      config.expires_field => expires_at(options) || nil)
+        when Float, Integer
+          { '_id' => key,
+            config.type_field => 'Number',
+            config.value_field => value,
+            # expires_field must be a Time object (BSON date datatype)
+            config.expires_field => expires_at(options) || nil }
+        when String
+          intvalue = value.to_i
+          { '_id' => key,
+            config.type_field => 'String',
+            config.value_field => intvalue.to_s == value ? intvalue : to_binary(value),
+            # @expires_field must be a Time object (BSON date datatype)
+            config.expires_field => expires_at(options) || nil }
+        else
+          raise ArgumentError, "Invalid value type: #{value.class}"
+        end
+      end
+
+      # BSON will use String#force_encoding to make the string 8-bit
+      # ASCII.  This could break unicode text so we should dup in this
+      # case, and it also fails with frozen strings.
+      def to_binary(str)
+        str = str.dup if str.frozen? || str.encoding != Encoding::ASCII_8BIT
+        ::BSON::Binary.new(str)
+      end
+
+      def from_binary(binary)
+        binary.is_a?(::BSON::Binary) ? binary.data : binary.to_s
+      end
+
+      def not_expired
+        {
+          :$or => [
+            { config.expires_field => nil },
+            { config.expires_field => { :$gte => Time.now } }
+          ]
+        }
+      end
+
+      def update_expiry(options, default)
+        if (expires = expires_at(options, default)) != nil
+          yield(expires || nil)
+        end
+      end
     end
   end
 end

data/lib/moneta/adapters/null.rb

@@ -6,8 +6,7 @@ module Moneta
       include Defaults
 
       # @param [Hash] options Options hash
-      def initialize(options = {})
-      end
+      def initialize(options = {}); end
 
       # (see Proxy#key?)
       def key?(key, options = {})

data/lib/moneta/adapters/pstore.rb

@@ -5,60 +5,72 @@ module Moneta
   module Adapters
     # PStore backend
     # @api public
-    class PStore
-      include Defaults
+    class PStore < Adapter
+      include NilValues
 
-      supports :create, :increment
-      attr_reader :backend
+      supports :create, :increment, :each_key
+
+      backend do |file:, threadsafe: false|
+        FileUtils.mkpath(::File.dirname(file))
+        ::PStore.new(file, threadsafe)
+      end
 
       # @param [Hash] options
       # @option options [String] :file PStore file
+      # @option options [Boolean] :threadsafe Makes the PStore thread-safe
       # @option options [::PStore] :backend Use existing backend instance
       def initialize(options = {})
-        @backend = options[:backend] ||
-          begin
-            raise ArgumentError, 'Option :file is required' unless options[:file]
-            FileUtils.mkpath(::File.dirname(options[:file]))
-            new_store(options)
-          end
+        super
+        @id = "Moneta::Adapters::PStore(#{object_id})"
       end
 
       # (see Proxy#key?)
       def key?(key, options = {})
-        @backend.transaction(true) { @backend.root?(key) }
+        transaction(true) { backend.root?(key) }
+      end
+
+      # (see Proxy#each_key)
+      def each_key(&block)
+        return enum_for(:each_key) { transaction(true) { backend.roots.size } } unless block_given?
+
+        transaction(true) do
+          backend.roots.each { |k| yield(k) }
+        end
+        self
       end
 
       # (see Proxy#load)
       def load(key, options = {})
-        @backend.transaction(true) { @backend[key] }
+        transaction(true) { backend[key] }
       end
 
       # (see Proxy#store)
       def store(key, value, options = {})
-        @backend.transaction { @backend[key] = value }
+        transaction { backend[key] = value }
       end
 
       # (see Proxy#delete)
       def delete(key, options = {})
-        @backend.transaction { @backend.delete(key) }
+        transaction { backend.delete(key) }
       end
 
       # (see Proxy#increment)
       def increment(key, amount = 1, options = {})
-        @backend.transaction do
-          value = Utils.to_int(@backend[key]) + amount
-          @backend[key] = value.to_s
+        transaction do
+          existing = backend[key]
+          value = (existing == nil ? 0 : Integer(existing)) + amount
+          backend[key] = value.to_s
           value
         end
       end
 
       # (see Proxy#create)
       def create(key, value, options = {})
-        @backend.transaction do
-          if @backend.root?(key)
+        transaction do
+          if backend.root?(key)
             false
           else
-            @backend[key] = value
+            backend[key] = value
             true
           end
         end
@@ -66,18 +78,49 @@ module Moneta
 
       # (see Proxy#clear)
       def clear(options = {})
-        @backend.transaction do
-          @backend.roots.each do |key|
-            @backend.delete(key)
+        transaction do
+          backend.roots.each do |key|
+            backend.delete(key)
           end
         end
         self
       end
 
+      # (see Proxy#values_at)
+      def values_at(*keys, **options)
+        transaction(true) { super }
+      end
+
+      def fetch_values(*keys, **options)
+        transaction(true) { super }
+      end
+
+      def slice(*keys, **options)
+        transaction(true) { super }
+      end
+
+      def merge!(pairs, options = {})
+        transaction { super }
+      end
+
       protected
 
-      def new_store(options)
-        ::PStore.new(options[:file], options[:threadsafe])
+      class TransactionError < StandardError; end
+
+      def transaction(read_only = false)
+        case Thread.current[@id]
+        when read_only, false
+          yield
+        when true
+          raise TransactionError, "Attempt to start read-write transaction inside a read-only transaction"
+        else
+          begin
+            Thread.current[@id] = read_only
+            backend.transaction(read_only) { yield }
+          ensure
+            Thread.current[@id] = nil
+          end
+        end
       end
     end
   end

data/lib/moneta/adapters/redis.rb

@@ -4,46 +4,52 @@ module Moneta
   module Adapters
     # Redis backend
     # @api public
-    class Redis
-      include Defaults
+    class Redis < Adapter
       include ExpiresSupport
 
-      supports :create, :increment
-      attr_reader :backend
+      supports :create, :increment, :each_key
 
-      # @param [Hash] options
-      # @option options [Integer] :expires Default expiration time
-      # @option options [::Redis] :backend Use existing backend instance
-      # @option options Other options passed to `Redis#new`
-      def initialize(options = {})
-        self.default_expires = options.delete(:expires)
-        @backend = options[:backend] || ::Redis.new(options)
-      end
+      # @!method initialize(options = {})
+      #   @param [Hash] options
+      #   @option options [Integer] :expires Default expiration time
+      #   @option options [::Redis] :backend Use existing backend instance
+      #   @option options Other options passed to `Redis#new`
+      backend { |**options| ::Redis.new(options) }
 
       # (see Proxy#key?)
       #
       # This method considers false and 0 as "no-expire" and every positive
       # number as a time to live in seconds.
       def key?(key, options = {})
-        if @backend.exists(key)
-          update_expires(key, options, nil)
-          true
-        else
-          false
+        with_expiry_update(key, default: nil, **options) do |pipeline_handle|
+          if pipeline_handle.respond_to?(:exists?)
+            pipeline_handle.exists?(key)
+          else
+            pipeline_handle.exists(key)
+          end
         end
       end
 
+      # (see Proxy#each_key)
+      def each_key(&block)
+        return enum_for(:each_key) unless block_given?
+
+        @backend.scan_each { |k| yield(k) }
+        self
+      end
+
       # (see Proxy#load)
       def load(key, options = {})
-        value = @backend.get(key)
-        update_expires(key, options, nil)
-        value
+        with_expiry_update(key, default: nil, **options) do |pipeline_handle|
+          pipeline_handle.get(key)
+        end
       end
 
       # (see Proxy#store)
       def store(key, value, options = {})
         if expires = expires_value(options)
-          @backend.setex(key, expires, value)
+          Numeric === expires and expires = (expires * 1000).to_i
+          @backend.psetex(key, expires, value)
         else
           @backend.set(key, value)
         end
@@ -52,17 +58,19 @@ module Moneta
 
       # (see Proxy#delete)
       def delete(key, options = {})
-        if value = load(key, options)
-          @backend.del(key)
-          value
+        future = nil
+        @backend.pipelined do |pipeline|
+          future = pipeline.get(key)
+          pipeline.del(key)
         end
+        future.value
       end
 
       # (see Proxy#increment)
       def increment(key, amount = 1, options = {})
-        value = @backend.incrby(key, amount)
-        update_expires(key, options)
-        value
+        with_expiry_update(key, **options) do |pipeline_handle|
+          pipeline_handle.incrby(key, amount)
+        end
       end
 
       # (see Proxy#clear)
@@ -73,8 +81,10 @@ module Moneta
 
       # (see Defaults#create)
       def create(key, value, options = {})
+        expires = expires_value(options, config.expires)
+
         if @backend.setnx(key, value)
-          update_expires(key, options)
+          update_expires(@backend, key, expires)
           true
         else
           false
@@ -87,14 +97,73 @@ module Moneta
         nil
       end
 
+      # (see Defaults#values_at)
+      def values_at(*keys, **options)
+        with_expiry_update(*keys, default: nil, **options) do |pipeline_handle|
+          pipeline_handle.mget(*keys)
+        end
+      end
+
+      # (see Defaults#merge!)
+      def merge!(pairs, options = {})
+        keys = pairs.map { |key, _| key }
+
+        if block_given?
+          old_values = @backend.mget(*keys)
+          updates = pairs.each_with_index.with_object({}) do |(pair, i), updates|
+            old_value = old_values[i]
+            if old_value != nil
+              key, new_value = pair
+              updates[key] = yield(key, old_value, new_value)
+            end
+          end
+          unless updates.empty?
+            pairs = if pairs.respond_to?(:merge)
+                      pairs.merge(updates)
+                    else
+                      Hash[pairs.to_a].merge!(updates)
+                    end
+          end
+        end
+
+        with_expiry_update(*keys, **options) do |pipeline_handle|
+          pipeline_handle.mset(*pairs.to_a.flatten(1))
+        end
+
+        self
+      end
+
       protected
 
-      def update_expires(key, options, default = @default_expires)
-        case expires = expires_value(options, default)
+      def update_expires(pipeline_handle, key, expires)
+        case expires
         when false
-          @backend.persist(key)
+          pipeline_handle.persist(key)
         when Numeric
-          @backend.expire(key, expires)
+          pipeline_handle.pexpire(key, (expires * 1000).to_i)
+        end
+      end
+
+      def with_expiry_update(*keys, default: config.expires, **options)
+        expires = expires_value(options, default)
+        if expires == nil
+          yield(@backend)
+        else
+          future = nil
+          @backend.multi do |pipeline|
+            # as of redis 4.6 calling redis methods on the redis client itself
+            # is deprecated in favor of a pipeline handle provided by the
+            # +multi+ call. This will cause in error in redis >= 5.0.
+            #
+            # In order to continue supporting redis versions < 4.6, the following
+            # fallback has been introduced and can be removed once moneta
+            # no longer supports redis < 4.6.
+
+            pipeline_handle = pipeline || @backend
+            future = yield(pipeline_handle)
+            keys.each { |key| update_expires(pipeline_handle, key, expires) }
+          end
+          future.value
         end
       end
     end

data/lib/moneta/adapters/restclient.rb

@@ -4,46 +4,47 @@ module Moneta
   module Adapters
     # Moneta rest client backend which works together with {Rack::MonetaRest}
     # @api public
-    class RestClient
-      include Defaults
-
-      attr_reader :backend
-
-      # @param [Hash] options
-      # @option options [String] :url URL
-      # @option options [Faraday connection] :backend Use existing backend instance
-      def initialize(options = {})
-        raise ArgumentError, 'Option :url is required' unless url = options[:url]
-        @backend = options[:backend] || ::Faraday.new(url: url)
+    class RestClient < Adapter
+      # @!method initialize(options = {})
+      #   @param [Hash] options
+      #   @option options [String] :url URL
+      #   @option options [Symbol] :adapter The adapter to tell Faraday to use
+      #   @option options [Faraday::Connection] :backend Use existing backend instance
+      #   @option options Other options passed to {Faraday::new} (unless
+      #     :backend option is provided).
+      backend do |url:, adapter: nil, **options|
+        ::Faraday.new(url, options) do |faraday|
+          faraday.adapter adapter if adapter
+        end
       end
 
       # (see Proxy#key?)
       def key?(key, options = {})
-        @backend.head(key).status == 200
+        backend.head(key).status == 200
       end
 
       # (see Proxy#load)
       def load(key, options = {})
-        response = @backend.get(key)
+        response = backend.get(key)
         response.status == 200 ? response.body : nil
       end
 
       # (see Proxy#store)
       def store(key, value, options = {})
-        response = @backend.post(key, value)
+        response = backend.post(key, value)
         raise "HTTP error #{response.status}" unless response.status == 200
         value
       end
 
       # (see Proxy#delete)
       def delete(key, options = {})
-        response = @backend.delete(key)
+        response = backend.delete(key)
         response.status == 200 ? response.body : nil
       end
 
       # (see Proxy#clear)
       def clear(options = {})
-        @backend.delete ''
+        backend.delete ''
         self
       end
     end