moneta 1.0.0 → 1.5.1

data/lib/moneta/adapters/cassandra.rb

@@ -4,61 +4,94 @@ module Moneta
   module Adapters
     # Cassandra backend
     # @api public
-    # @author Potapov Sergey (aka Blake)
-    class Cassandra
-      include Defaults
+    class Cassandra < Adapter
       include ExpiresSupport
 
-      attr_reader :backend
+      supports :each_key
+
+      config :table, default: 'moneta'
+      config :key_column, default: 'key'
+      config :value_column, default: 'value'
+      config :updated_column, default: 'updated_at'
+      config :expired_column, default: 'expired'
+      config :read_consistency, default: :all
+      config :write_consistency, default: :all
+
+      backend do |keyspace: 'moneta', cluster: nil, create_keyspace: nil, **options|
+        cluster ||= ::Cassandra.cluster(options).tap do |own_cluster|
+          @own_cluster = own_cluster
+        end
+
+        begin
+          cluster.connect(keyspace)
+        rescue ::Cassandra::Errors::InvalidError
+          backend = cluster.connect
+          create_keyspace(backend, keyspace, create_keyspace)
+          backend.execute("USE " + keyspace)
+          backend
+        end
+      end
 
       # @param [Hash] options
       # @option options [String] :keyspace ('moneta') Cassandra keyspace
-      # @option options [String] :column_family ('moneta') Cassandra column family
+      # @option options [String] :table ('moneta') Cassandra table
       # @option options [String] :host ('127.0.0.1') Server host name
       # @option options [Integer] :port (9160) Server port
       # @option options [Integer] :expires Default expiration time
-      # @option options [::Cassandra] :backend Use existing backend instance
-      # @option options Other options passed to `Cassandra#new`
+      # @option options [String] :key_column ('key') Name of the key column
+      # @option options [String] :value_column ('value') Name of the value
+      #   column
+      # @option options [String] :updated_column ('updated_at') Name of the
+      #   column used to track last update
+      # @option options [String] :expired_column ('expired') Name of the column
+      #   used to track expiry
+      # @option options [Symbol] :read_consistency (:all) Default read
+      #   consistency
+      # @option options [Symbol] :write_consistency (:all) Default write
+      #   consistency
+      # @option options [Proc, Boolean, Hash] :create_keyspace Provide a proc
+      #   for creating the keyspace, or a Hash of options to use when creating
+      #   it, or set to false to disable.  The Proc will only be called if the
+      #   keyspace does not already exist.
+      # @option options [::Cassandra::Cluster] :cluster Existing cluster to use
+      # @option options [::Cassandra::Session] :backend Existing session to use
+      # @option options Other options passed to `Cassandra#cluster`
       def initialize(options = {})
-        self.default_expires = options.delete(:expires)
-        @cf = (options.delete(:column_family) || 'moneta').to_sym
-        if options[:backend]
-          @backend = options[:backend]
-        else
-          keyspace = options.delete(:keyspace) || 'moneta'
-          options[:retries] ||= 3
-          options[:connect_timeout] ||= 10
-          options[:timeout] ||= 10
-          @backend = ::Cassandra.new('system',
-                                     "#{options[:host] || '127.0.0.1'}:#{options[:port] || 9160}",
-                                     options)
-          unless @backend.keyspaces.include?(keyspace)
-            cf_def = ::Cassandra::ColumnFamily.new(keyspace: keyspace, name: @cf.to_s)
-            ks_def = ::Cassandra::Keyspace.new(name: keyspace,
-                                               strategy_class: 'SimpleStrategy',
-                                               strategy_options: { 'replication_factor' => '1' },
-                                               replication_factor: 1,
-                                               cf_defs: [cf_def])
-            # Wait for keyspace to be created (issue #24)
-            10.times do
-              begin
-                @backend.add_keyspace(ks_def)
-              rescue Exception => ex
-                warn "Moneta::Adapters::Cassandra - #{ex.message}"
-              end
-              break if @backend.keyspaces.include?(keyspace)
-              sleep 0.1
-            end
-          end
-          @backend.keyspace = keyspace
-        end
+        super
+
+        backend.execute <<-CQL
+          CREATE TABLE IF NOT EXISTS #{config.table} (
+            #{config.key_column} blob,
+            #{config.value_column} blob,
+            #{config.updated_column} timeuuid,
+            #{config.expired_column} boolean,
+            PRIMARY KEY (#{config.key_column}, #{config.updated_column})
+          )
+        CQL
+
+        prepare_statements
       end
 
       # (see Proxy#key?)
       def key?(key, options = {})
-        if @backend.exists?(@cf, key)
-          load(key, options) if options.include?(:expires)
+        rc, wc = consistency(options)
+        if (expires = expires_value(options, nil)) != nil
+          # Because Cassandra expires each value in a column, rather than the
+          # whole column, when we want to update the expiry we load the value
+          # and then re-set it in order to update the TTL.
+          return false unless
+            row = backend.execute(@load, options.merge(consistency: rc, arguments: [key])).first and
+              row[config.expired_column] != nil
+          backend.execute(@update_expires,
+                          options.merge(consistency: wc,
+                                        arguments: [(expires || 0).to_i,
+                                                    timestamp,
+                                                    row[config.value_column],
+                                                    key,
+                                                    row[config.updated_column]]))
           true
+        elsif row = backend.execute(@key, options.merge(consistency: rc, arguments: [key])).first
+          row[config.expired_column] != nil
         else
           false
         end
@@ -66,38 +99,257 @@ module Moneta
 
       # (see Proxy#load)
       def load(key, options = {})
-        if value = @backend.get(@cf, key)
-          expires = expires_value(options, nil)
-          @backend.insert(@cf, key, {'value' => value['value'] }, ttl: expires || nil) if expires != nil
-          value['value']
+        rc, wc = consistency(options)
+        if row = backend.execute(@load, options.merge(consistency: rc, arguments: [key])).first and row[config.expired_column] != nil
+          if (expires = expires_value(options, nil)) != nil
+            backend.execute(@update_expires,
+                            options.merge(consistency: wc,
+                                          arguments: [(expires || 0).to_i,
+                                                      timestamp,
+                                                      row[config.value_column],
+                                                      key,
+                                                      row[config.updated_column]]))
+          end
+          row[config.value_column]
         end
       end
 
       # (see Proxy#store)
       def store(key, value, options = {})
-        @backend.insert(@cf, key, {'value' => value}, ttl: expires_value(options) || nil)
+        _, wc = consistency(options)
+        expires = expires_value(options)
+        t = timestamp
+        batch = backend.batch do |batch|
+          batch.add(@store_delete, arguments: [t, key])
+          batch.add(@store, arguments: [key, value, (expires || 0).to_i, t + 1])
+        end
+        backend.execute(batch, options.merge(consistency: wc))
         value
       end
 
       # (see Proxy#delete)
       def delete(key, options = {})
-        if value = load(key, options)
-          @backend.remove(@cf, key)
-          value
+        rc, wc = consistency(options)
+        result = backend.execute(@delete_value, options.merge(consistency: rc, arguments: [key]))
+        if row = result.first and row[config.expired_column] != nil
+          backend.execute(@delete, options.merge(consistency: wc, arguments: [timestamp, key, row[config.updated_column]]))
+          row[config.value_column]
         end
       end
 
       # (see Proxy#clear)
       def clear(options = {})
-        @backend.clear_column_family!(@cf)
+        backend.execute(@clear)
         self
       end
 
       # (see Proxy#close)
       def close
-        @backend.disconnect!
+        backend.close_async
+        @backend = nil
+        if @own_cluster
+          @own_cluster.close_async
+          @own_cluster = nil
+        end
         nil
       end
+
+      # (see Proxy#each_key)
+      def each_key
+        rc, = consistency
+        return enum_for(:each_key) unless block_given?
+        result = backend.execute(@each_key, consistency: rc, page_size: 100)
+        loop do
+          result.each do |row|
+            next if row[config.expired_column] == nil
+            yield row[config.key_column]
+          end
+
+          break if result.last_page?
+          result = result.next_page
+        end
+        self
+      end
+
+      # (see Proxy#slice)
+      def slice(*keys, **options)
+        rc, wc = consistency(options)
+        result = backend.execute(@slice, options.merge(consistency: rc, arguments: [keys]))
+        expires = expires_value(options, nil)
+        updated = [] if expires != nil
+        pairs = result.map do |row|
+          next if row[config.expired_column] == nil
+          if expires != nil
+            updated << [row[config.key_column], row[config.value_column], row[config.updated_column]]
+          end
+          [row[config.key_column], row[config.value_column]]
+        end.compact
+
+        if expires != nil && !updated.empty?
+          ttl = (expires || 0).to_i
+          t = timestamp
+          batch = backend.batch do |batch|
+            updated.each do |key, value, updated|
+              batch.add(@update_expires, arguments: [ttl, t, value, key, updated])
+            end
+          end
+
+          backend.execute(batch, options.merge(consistency: wc))
+        end
+
+        pairs
+      end
+
+      # (see Proxy#values_at)
+      def values_at(*keys, **options)
+        hash = Hash[slice(*keys, **options)]
+        keys.map { |key| hash[key] }
+      end
+
+      # (see Proxy#fetch_values)
+      def fetch_values(*keys, **options)
+        return values_at(*keys, **options) unless block_given?
+        hash = Hash[slice(*keys, **options)]
+        keys.map do |key|
+          if hash.key?(key)
+            hash[key]
+          else
+            yield key
+          end
+        end
+      end
+
+      # (see Proxy#merge!)
+      def merge!(pairs, options = {})
+        keys = pairs.map { |k, _| k }.to_a
+        return self if keys.empty?
+
+        if block_given?
+          existing = Hash[slice(*keys, **options)]
+          pairs = pairs.map do |key, new_value|
+            if existing.key?(key)
+              [key, yield(key, existing[key], new_value)]
+            else
+              [key, new_value]
+            end
+          end
+        end
+
+        _rc, wc = consistency(options)
+        expires = expires_value(options)
+        t = timestamp
+        batch = backend.batch do |batch|
+          batch.add(@merge_delete, arguments: [t, keys])
+          pairs.each do |key, value|
+            batch.add(@store, arguments: [key, value, (expires || 0).to_i, t + 1])
+          end
+        end
+        backend.execute(batch, options.merge(consistency: wc))
+
+        self
+      end
+
+      private
+
+      def timestamp
+        (Time.now.to_r * 1_000_000).to_i
+      end
+
+      def create_keyspace(backend, keyspace, create_keyspace)
+        options = {
+          replication: {
+            class: 'SimpleStrategy',
+            replication_factor: 1
+          }
+        }
+
+        case create_keyspace
+        when Proc
+          return create_keyspace.call(keyspace)
+        when false
+          return
+        when Hash
+          options.merge!(create_keyspace)
+        end
+
+        # This is a bit hacky, but works.  Options in Cassandra look like JSON,
+        # but use single quotes instead of double-quotes.
+        require 'multi_json'
+        option_str = options.map do |key, value|
+          key.to_s + ' = ' + MultiJson.dump(value).tr(?", ?')
+        end.join(' AND ')
+
+        backend.execute "CREATE KEYSPACE IF NOT EXISTS %<keyspace>s WITH %<options>s" % {
+          keyspace: keyspace,
+          options: option_str
+        }
+      rescue ::Cassandra::Errors::TimeoutError
+        tries ||= 0
+        (tries += 1) <= 3 ? retry : raise
+      end
+
+      def prepare_statements
+        @key = backend.prepare(<<-CQL)
+          SELECT #{config.updated_column}, #{config.expired_column}
+          FROM #{config.table} WHERE #{config.key_column} = ?
+          LIMIT 1
+        CQL
+        @store_delete = backend.prepare(<<-CQL)
+          DELETE FROM #{config.table}
+          USING TIMESTAMP ?
+          WHERE #{config.key_column} = ?
+        CQL
+        @store = backend.prepare(<<-CQL)
+          INSERT INTO #{config.table} (#{config.key_column}, #{config.value_column}, #{config.updated_column}, #{config.expired_column})
+          VALUES (?, ?, now(), false)
+          USING TTL ? AND TIMESTAMP ?
+        CQL
+        @load = backend.prepare(<<-CQL)
+          SELECT #{config.value_column}, #{config.updated_column}, #{config.expired_column}
+          FROM #{config.table}
+          WHERE #{config.key_column} = ?
+          LIMIT 1
+        CQL
+        @update_expires = backend.prepare(<<-CQL)
+          UPDATE #{config.table}
+          USING TTL ? AND TIMESTAMP ?
+          SET #{config.value_column} = ?, #{config.expired_column} = false
+          WHERE #{config.key_column} = ? AND #{config.updated_column} = ?
+        CQL
+        @clear = backend.prepare("TRUNCATE #{config.table}")
+        @delete_value = backend.prepare(<<-CQL)
+          SELECT #{config.value_column}, #{config.updated_column}, #{config.expired_column}
+          FROM #{config.table}
+          WHERE #{config.key_column} = ?
+          LIMIT 1
+        CQL
+        @delete = backend.prepare(<<-CQL, idempotent: true)
+          DELETE FROM #{config.table}
+          USING TIMESTAMP ?
+          WHERE #{config.key_column} = ? AND #{config.updated_column} = ?
+        CQL
+        @each_key = backend.prepare(<<-CQL)
+          SELECT #{config.key_column}, #{config.expired_column}
+          FROM #{config.table}
+        CQL
+        @slice = backend.prepare(<<-CQL)
+          SELECT #{config.key_column}, #{config.value_column}, #{config.updated_column}, #{config.expired_column}
+          FROM #{config.table}
+          WHERE #{config.key_column} IN ?
+        CQL
+        @merge_delete = backend.prepare(<<-CQL)
+          DELETE FROM #{config.table}
+          USING TIMESTAMP ?
+          WHERE #{config.key_column} IN ?
+        CQL
+      end
+
+      def consistency(options = {})
+        [
+          options[:read_consistency] || config.read_consistency,
+          options[:write_consistency] || config.write_consistency
+        ]
+      end
     end
   end
 end

data/lib/moneta/adapters/client.rb

@@ -4,74 +4,108 @@ module Moneta
   module Adapters
     # Moneta client backend
     # @api public
-    class Client
-      include Defaults
-
-      # @param [Hash] options
-      # @option options [Integer] :port (9000) TCP port
-      # @option options [String] :host ('127.0.0.1') Hostname
-      # @option options [String] :socket Unix socket file name as alternative to `:port` and `:host`
-      def initialize(options = {})
-        @socket = options[:socket] ? UNIXSocket.open(options[:socket]) :
-          TCPSocket.open(options[:host] || '127.0.0.1', options[:port] || 9000)
+    class Client < Adapter
+      # @!method initialize(options = {})
+      #   @param [Hash] options
+      #   @option options [TCPSocket | UNIXSocket] :backend an open socket to use
+      #   @option options [Integer] :port (9000) TCP port
+      #   @option options [String] :host ('127.0.0.1') Hostname
+      #   @option options [String] :socket Unix socket file name as alternative to `:port` and `:host`
+      backend do |socket: nil, host: '127.0.0.1', port: 9000|
+        if socket
+          UNIXSocket.open(socket)
+        else
+          TCPSocket.open(host, port)
+        end
       end
 
       # (see Proxy#key?)
       def key?(key, options = {})
         write(:key?, key, options)
-        read
+        read_msg
       end
 
       # (see Proxy#load)
       def load(key, options = {})
         write(:load, key, options)
-        read
+        read_msg
       end
 
       # (see Proxy#store)
       def store(key, value, options = {})
         write(:store, key, value, options)
-        read
+        read_msg
         value
       end
 
       # (see Proxy#delete)
       def delete(key, options = {})
         write(:delete, key, options)
-        read
+        read_msg
       end
 
       # (see Proxy#increment)
       def increment(key, amount = 1, options = {})
         write(:increment, key, amount, options)
-        read
+        read_msg
       end
 
       # (see Proxy#create)
       def create(key, value, options = {})
         write(:create, key, value, options)
-        read
+        read_msg
       end
 
       # (see Proxy#clear)
       def clear(options = {})
         write(:clear, options)
-        read
+        read_msg
         self
       end
 
       # (see Proxy#close)
       def close
-        @socket.close
+        backend.close
         nil
       end
 
+      # (see Proxy#each_key)
+      def each_key
+        raise NotImplementedError, 'each_key is not supported' unless supports?(:each_key)
+        return enum_for(:each_key) unless block_given?
+
+        begin
+          write(:each_key)
+          yield_break = false
+
+          loop do
+            write('NEXT')
+
+            # A StopIteration error will be raised by this call if the server
+            # reached the end of the enumeration.  This will stop the loop
+            # automatically.
+            result = read_msg
+
+            # yield_break will be true in the ensure block (below) if anything
+            # happened during the yield to stop further enumeration.
+            yield_break = true
+            yield result
+            yield_break = false
+          end
+        ensure
+          write('BREAK') if yield_break
+          read_msg # nil return from each_key
+        end
+
+        self
+      end
+
       # (see Default#features)
       def features
         @features ||=
           begin
             write(:features)
-            read.freeze
+            read_msg.freeze
           end
       end
 
@@ -79,12 +113,27 @@ module Moneta
 
       def write(*args)
         s = Marshal.dump(args)
-        @socket.write([s.bytesize].pack('N') << s)
+        backend.write([s.bytesize].pack('N') << s)
+      end
+
+      # JRuby doesn't support socket#recv with flags
+      if defined?(JRUBY_VERSION)
+        def read(bytes)
+          received = backend.read(bytes)
+          raise EOFError, "Server closed socket" unless received && received.bytesize == bytes
+          received
+        end
+      else
+        def read(bytes)
+          received = backend.recv(bytes, Socket::MSG_WAITALL)
+          raise EOFError, "Server closed socket" unless received && received.bytesize == bytes
+          received
+        end
       end
 
-      def read
-        size = @socket.read(4).unpack('N').first
-        result = Marshal.load(@socket.read(size))
+      def read_msg
+        size = read(4).unpack('N').first
+        result = Marshal.load(read(size))
         raise result if Exception === result
         result
       end