aerospike 2.13.0 → 2.18.0

data/lib/aerospike/cdt/map_order.rb

@@ -20,15 +20,15 @@ module Aerospike
 
       ##
       # Map is not ordered. This is the default.
-      UNORDERED = 0
+      UNORDERED = {attr: 0, flag: 0x40}
 
       ##
       # Order map by key.
-      KEY_ORDERED = 1
+      KEY_ORDERED = {attr: 1, flag: 0x80}
 
       ##
       # Order map by key, then value.
-      KEY_VALUE_ORDERED = 3
+      KEY_VALUE_ORDERED = {attr: 3, flag: 0xc0}
 
       ##
       # Default order

data/lib/aerospike/client.rb

@@ -44,6 +44,8 @@ module Aerospike
     attr_accessor :default_read_policy
     attr_accessor :default_scan_policy
     attr_accessor :default_write_policy
+    attr_accessor :default_operate_policy
+    attr_accessor :cluster
 
     def initialize(hosts = nil, policy: ClientPolicy.new, connect: true)
 
@@ -224,8 +226,19 @@ module Aerospike
     def truncate(namespace, set_name = nil, before_last_update = nil, options = {})
       policy = create_policy(options, Policy, default_info_policy)
 
-      str_cmd = "truncate:namespace=#{namespace}"
-      str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
+      node = @cluster.random_node
+      conn = node.get_connection(policy.timeout)
+
+      if set_name && !set_name.to_s.strip.empty?
+        str_cmd = "truncate:namespace=#{namespace}"
+        str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
+      else
+        if node.supports_feature?(Aerospike::Features::TRUNCATE_NAMESPACE)
+          str_cmd = "truncate-namespace:namespace=#{namespace}"
+        else
+          str_cmd = "truncate:namespace=#{namespace}"
+        end
+      end
 
       if before_last_update
         lut_nanos = (before_last_update.to_f * 1_000_000_000.0).round
@@ -235,8 +248,7 @@ module Aerospike
         str_cmd << ";lut=now"
       end
 
-      # Send index command to one node. That node will distribute the command to other nodes.
-      response = send_info_command(policy, str_cmd).upcase
+      response = send_info_command(policy, str_cmd, node).upcase
       return if response == 'OK'
       raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_ERROR, "Truncate failed: #{response}")
     end
@@ -372,7 +384,7 @@ module Aerospike
     #  read the result, all in one database call. Operations are executed in
     #  the order they are specified.
     def operate(key, operations, options = nil)
-      policy = create_policy(options, WritePolicy, default_write_policy)
+      policy = create_policy(options, OperatePolicy, default_operate_policy)
 
       command = OperateCommand.new(@cluster, policy, key, operations)
       execute_command(command)
@@ -816,11 +828,16 @@ module Aerospike
       self.default_query_policy = create_policy(policies[:query], QueryPolicy)
       self.default_scan_policy = create_policy(policies[:scan], ScanPolicy)
       self.default_write_policy = create_policy(policies[:write], WritePolicy)
+      self.default_operate_policy = create_policy(policies[:operate], OperatePolicy)
     end
 
-    def send_info_command(policy, command)
+    def send_info_command(policy, command, node = nil)
       Aerospike.logger.debug { "Sending info command: #{command}" }
-      _, response = @cluster.request_info(policy, command).first
+      if node
+        _, response = @cluster.request_node_info(node, policy, command).first
+      else
+        _, response = @cluster.request_info(policy, command).first
+      end
       response.to_s
     end
 

data/lib/aerospike/cluster.rb

@@ -28,6 +28,7 @@ module Aerospike
     attr_reader :features, :tls_options
     attr_reader :cluster_id, :aliases
     attr_reader :cluster_name
+    attr_accessor :rack_aware, :rack_id
 
     def initialize(policy, hosts)
       @cluster_seeds = hosts
@@ -37,6 +38,8 @@ module Aerospike
       @tend_interval = policy.tend_interval
       @cluster_name = policy.cluster_name
       @tls_options = policy.tls
+      @rack_aware = policy.rack_aware
+      @rack_id = policy.rack_id
 
       @replica_index = Atomic.new(0)
 
@@ -111,6 +114,8 @@ module Aerospike
           return master_node(partition)
         when Aerospike::Replica::MASTER_PROLES
           return master_proles_node(partition)
+        when Aerospike::Replica::PREFER_RACK
+          return rack_node(partition, seq)
         when Aerospike::Replica::RANDOM
           return random_node
         else
@@ -125,6 +130,8 @@ module Aerospike
           return master_node(partition)
         when Aerospike::Replica::MASTER_PROLES
           return master_proles_node(partition)
+        when Aerospike::Replica::PREFER_RACK
+          return rack_node(partition, seq)
         when Aerospike::Replica::SEQUENCE
           return sequence_node(partition, seq)
         when Aerospike::Replica::RANDOM
@@ -149,6 +156,38 @@ module Aerospike
       node
     end
 
+    # Returns a node on the cluster
+    def rack_node(partition, seq)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      is_retry = seq.value > -1
+
+      node = nil
+      fallback = nil
+      for i in 1..replica_array.length
+        idx = (seq.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
+
+        next if !node
+
+        fallback = node
+
+        # If fallback exists, do not retry on node where command failed,
+        # even if fallback is not on the same rack.
+        return fallback if is_retry && fallback && i == replica_array.length
+
+        return node if node && node.active? && node.has_rack(partition.namespace, @rack_id)
+      end
+
+      return fallback if fallback
+
+      raise Aerospike::Exceptions::InvalidNode
+    end
+
     # Returns a node on the cluster for read operations
     def master_proles_node(partition)
       partition_map = partitions
@@ -256,6 +295,13 @@ module Aerospike
       end
     end
 
+    def request_node_info(node, policy, *commands)
+      conn = node.get_connection(policy.timeout)
+      Info.request(conn, *commands).tap do
+        node.put_connection(conn)
+      end
+    end
+
     def supports_feature?(feature)
       @features.get.include?(feature.to_s)
     end
@@ -352,6 +398,7 @@ module Aerospike
 
       nodes.each do |node|
         node.refresh_partitions(peers) if node.partition_generation.changed?
+        node.refresh_racks if node.rebalance_generation.changed?
       end
 
       if peers.generation_changed? || !peers.use_peers?

data/lib/aerospike/cluster/rack_parser.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'base64'
+
+module Aerospike
+
+  class RackParser #:nodoc:
+
+    attr_accessor :rebalance_generation, :racks
+
+    REBALANCE_GENERATION = "rebalance-generation"
+    RACK_IDS = "rack-ids"
+
+    def initialize(node, conn)
+      @node = node
+      @conn = conn
+      @racks = nil
+    end
+
+    def update_racks
+      # Use low-level info methods and parse byte array directly for maximum performance.
+      # Receive format: rack-ids\t
+      #                 <ns1>:<rack-id>...;
+      #                 <ns2>:<rack-id>...; ...
+      info_map = Info.request(@conn, REBALANCE_GENERATION, RACK_IDS)
+
+      @rebalance_generation = info_map[REBALANCE_GENERATION].to_i
+
+      info = info_map[RACK_IDS]
+      if !info || info.length == 0
+        raise Aerospike::Exceptions::Connection.new("#{RACK_IDS} response for node #{@node.name} is empty")
+      end
+
+      @buffer = info
+      @length = info.length
+      @offset = 0
+
+      while @offset < @length && @buffer[@offset] != '\n'
+        namespace = parse_name
+        rack_id = parse_rack_id
+
+        @racks = {} if !@racks
+        @racks[namespace] = rack_id
+      end
+
+      @racks
+    end
+
+    private
+
+    def parse_name
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ':'
+        @offset+=1
+      end
+
+      # Parse namespace.
+      namespace = @buffer[beginning...@offset].strip
+
+      if namespace.length <= 0 || namespace.length >= 32
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid rack namespace #{namespace}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      namespace
+    end
+
+    def parse_rack_id
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ';'
+        @offset+=1
+      end
+
+      # Parse rack_id
+      rack_id = @buffer[beginning...@offset].strip.to_i
+
+      if rack_id < 0
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid rack_id #{rack_id}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      rack_id
+    end
+
+    def get_truncated_response
+      max = @length
+      @length = max if @length > 200
+      @buffer[0...max]
+    end
+
+
+  end # class
+
+end # module

data/lib/aerospike/command/batch_direct_command.rb

@@ -75,6 +75,7 @@ module Aerospike
       end
 
       end_cmd
+      mark_compressed(@policy)
     end
 
     # Parse all results in the batch.  Add records to shared list.

data/lib/aerospike/command/batch_index_command.rb

@@ -98,6 +98,7 @@ module Aerospike
         end
       end
       end_cmd
+      mark_compressed(@policy)
     end
 
     # Parse all results in the batch.  Add records to shared list.

data/lib/aerospike/command/command.rb

@@ -16,6 +16,7 @@
 # the License.
 
 require 'time'
+require 'zlib'
 
 require 'msgpack'
 require 'aerospike/result_code'
@@ -42,6 +43,9 @@ module Aerospike
   # Involve all replicas in read operation.
   INFO1_CONSISTENCY_ALL = Integer(1 << 6)
 
+  # Tell server to compress it's response.
+  INFO1_COMPRESS_RESPONSE = (1 << 7)
+
   # Create or update record
   INFO2_WRITE = Integer(1 << 0)
   # Fling a record into the belly of Moloch.
@@ -55,6 +59,9 @@ module Aerospike
   # Create only. Fail if record already exists.
   INFO2_CREATE_ONLY = Integer(1 << 5)
 
+  # Return a result for every operation.
+  INFO2_RESPOND_ALL_OPS = Integer(1 << 7)
+
   # This is the last of a multi-part message.
   INFO3_LAST = Integer(1 << 0)
   # Commit to master only before declaring success.
@@ -72,8 +79,10 @@ module Aerospike
   OPERATION_HEADER_SIZE      = 8
   MSG_REMAINING_HEADER_SIZE  = 22
   DIGEST_SIZE                = 20
+  COMPRESS_THRESHOLD         = 128
   CL_MSG_VERSION             = 2
   AS_MSG_TYPE                = 3
+  AS_MSG_TYPE_COMPRESSED     = 4
 
   class Command #:nodoc:
 
@@ -83,6 +92,8 @@ module Aerospike
 
       @node = node
 
+      @compress = false
+
       # will add before use
       @sequence = Atomic.new(-1)
 
@@ -118,6 +129,7 @@ module Aerospike
       end
 
       end_cmd
+      mark_compressed(policy)
     end
 
     # Writes the command for delete operations
@@ -245,10 +257,12 @@ module Aerospike
       read_attr = 0
       write_attr = 0
       read_header = false
+      record_bin_multiplicity = policy.record_bin_multiplicity == RecordBinMultiplicity::ARRAY
 
       operations.each do |operation|
         case operation.op_type
-        when Aerospike::Operation::READ
+        when Aerospike::Operation::READ, Aerospike::Operation::CDT_READ, 
+              Aerospike::Operation::HLL_READ, Aerospike::Operation::BIT_READ
           read_attr |= INFO1_READ
 
           # Read all bins if no bin is specified.
@@ -262,17 +276,22 @@ module Aerospike
           read_attr |= INFO1_READ
           read_header = true
 
-        when Aerospike::Operation::CDT_READ
-          read_attr |= INFO1_READ
-
         else
           write_attr = INFO2_WRITE
         end
 
+        if [Aerospike::Operation::HLL_MODIFY, Aerospike::Operation::HLL_READ, 
+              Aerospike::Operation::BIT_MODIFY, Aerospike::Operation::BIT_READ].include?(operation.op_type)
+          record_bin_multiplicity = true
+        end
+
         estimate_operation_size_for_operation(operation)
       end
       size_buffer
 
+
+      write_attr |= INFO2_RESPOND_ALL_OPS if write_attr != 0 && record_bin_multiplicity
+
       if write_attr != 0
         write_header_with_policy(policy, read_attr, write_attr, field_count, operations.length)
       else
@@ -288,6 +307,7 @@ module Aerospike
       write_operation_for_bin(nil, Aerospike::Operation::READ) if read_header
 
       end_cmd
+      mark_compressed(policy)
     end
 
     def set_udf(policy, key, package_name, function_name, args)
@@ -310,6 +330,7 @@ module Aerospike
       write_field_bytes(arg_bytes, Aerospike::FieldType::UDF_ARGLIST)
 
       end_cmd
+      mark_compressed(policy)
     end
 
     def set_scan(policy, namespace, set_name, bin_names)
@@ -425,10 +446,14 @@ module Aerospike
           @node = get_node
           @conn = @node.get_connection(@policy.timeout)
         rescue => e
-          # Socket connection error has occurred. Decrease health and retry.
-          @node.decrease_health
+          if @node
+            # Socket connection error has occurred. Decrease health and retry.
+            @node.decrease_health
 
-          Aerospike.logger.error("Node #{@node.to_s}: #{e}")
+            Aerospike.logger.error("Node #{@node.to_s}: #{e}")
+          else
+            Aerospike.logger.error("No node available for transaction: #{e}")
+          end
           next
         end
 
@@ -579,6 +604,7 @@ module Aerospike
     # Generic header write.
     def write_header(policy, read_attr, write_attr, field_count, operation_count)
       read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
+      read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
 
       # Write all header data except total size which must be written last.
       @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -628,6 +654,7 @@ module Aerospike
       info_attr |= INFO3_COMMIT_MASTER if policy.commit_level == Aerospike::CommitLevel::COMMIT_MASTER
       read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
       write_attr |= INFO2_DURABLE_DELETE if policy.durable_delete
+      read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
 
       # Write all header data except total size which must be written last.
       @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -810,6 +837,48 @@ module Aerospike
       @data_buffer.write_int64(size, 0)
     end
 
+    def use_compression?
+      @compress
+    end
+
+    def compress_buffer
+      if @data_offset > COMPRESS_THRESHOLD
+        compressed = Zlib::deflate(@data_buffer.buf, Zlib::DEFAULT_COMPRESSION)
+
+        # write original size as header
+        proto_s = "%08d" % 0
+        proto_s[0, 8] = [@data_offset].pack('q>')
+        compressed.prepend(proto_s)
+
+        # write proto
+        proto = (compressed.size+8) | Integer(CL_MSG_VERSION << 56) | Integer(AS_MSG_TYPE << 48)
+        proto_s = "%08d" % 0
+        proto_s[0, 8] = [proto].pack('q>')
+        compressed.prepend(proto_s)
+
+        @data_buffer = Buffer.new(-1, compressed)
+        @data_offset = @data_buffer.size
+      end
+    end
+
+    # isCompressed returns the length of the compressed buffer.
+    # If the buffer is not compressed, the result will be -1
+    def compressed_size
+      # A number of these are commented out because we just don't care enough to read
+      # that section of the header. If we do care, uncomment and check!
+      proto = @data_buffer.read_int64(0)
+      size = proto & 0xFFFFFFFFFFFF
+      msg_type = (proto >> 48) & 0xFF
+
+      return nil if msg_type != AS_MSG_TYPE_COMPRESSED
+
+      size
+    end
+
+    def mark_compressed(policy)
+      @compress = policy.use_compression
+    end
+
   end # class
 
 end # module