aerospike 2.10.0 → 2.15.0

data/lib/aerospike/cdt/list_operation.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_operation.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_order.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2017 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_policy.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_return_type.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2017 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_write_mode.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/client.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -44,6 +44,8 @@ module Aerospike
     attr_accessor :default_read_policy
     attr_accessor :default_scan_policy
     attr_accessor :default_write_policy
+    attr_accessor :default_operate_policy
+    attr_accessor :cluster
 
     def initialize(hosts = nil, policy: ClientPolicy.new, connect: true)
 
@@ -224,8 +226,19 @@ module Aerospike
     def truncate(namespace, set_name = nil, before_last_update = nil, options = {})
       policy = create_policy(options, Policy, default_info_policy)
 
-      str_cmd = "truncate:namespace=#{namespace}"
-      str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
+      node = @cluster.random_node
+      conn = node.get_connection(policy.timeout)
+
+      if set_name && !set_name.to_s.strip.empty?
+        str_cmd = "truncate:namespace=#{namespace}"
+        str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
+      else
+        if node.supports_feature(Aerospike::Features::TRUNCATE_NAMESPACE)
+          str_cmd = "truncate-namespace:namespace=#{namespace}"
+        else
+          str_cmd = "truncate:namespace=#{namespace}"
+        end
+      end
 
       if before_last_update
         lut_nanos = (before_last_update.to_f * 1_000_000_000.0).round
@@ -235,8 +248,7 @@ module Aerospike
         str_cmd << ";lut=now"
       end
 
-      # Send index command to one node. That node will distribute the command to other nodes.
-      response = send_info_command(policy, str_cmd).upcase
+      response = send_info_command(policy, str_cmd, node).upcase
       return if response == 'OK'
       raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_ERROR, "Truncate failed: #{response}")
     end
@@ -322,11 +334,11 @@ module Aerospike
 
       if policy.use_batch_direct
         key_map = BatchItem.generate_map(keys)
-        execute_batch_direct_commands(keys) do |node, batch|
+        execute_batch_direct_commands(policy, keys) do |node, batch|
           BatchDirectCommand.new(node, batch, policy, key_map, bin_names, results, info_flags)
         end
       else
-        execute_batch_index_commands(keys) do |node, batch|
+        execute_batch_index_commands(policy, keys) do |node, batch|
           BatchIndexCommand.new(node, batch, policy, bin_names, results, info_flags)
         end
       end
@@ -351,11 +363,11 @@ module Aerospike
 
       if policy.use_batch_direct
         key_map = BatchItem.generate_map(keys)
-        execute_batch_direct_commands(keys) do |node, batch|
+        execute_batch_direct_commands(policy, keys) do |node, batch|
           BatchDirectExistsCommand.new(node, batch, policy, key_map, results)
         end
       else
-        execute_batch_index_commands(keys) do |node, batch|
+        execute_batch_index_commands(policy, keys) do |node, batch|
           BatchIndexExistsCommand.new(node, batch, policy, results)
         end
       end
@@ -372,7 +384,7 @@ module Aerospike
     #  read the result, all in one database call. Operations are executed in
     #  the order they are specified.
     def operate(key, operations, options = nil)
-      policy = create_policy(options, WritePolicy, default_write_policy)
+      policy = create_policy(options, OperatePolicy, default_operate_policy)
 
       command = OperateCommand.new(@cluster, policy, key, operations)
       execute_command(command)
@@ -526,7 +538,7 @@ module Aerospike
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing UDF failed because cluster is empty.")
       end
 
-      # TODO: wait until all migrations are finished
+      statement = statement.clone
       statement.set_aggregate_function(package_name, function_name, function_args, false)
 
       # Use a thread per node
@@ -816,11 +828,16 @@ module Aerospike
       self.default_query_policy = create_policy(policies[:query], QueryPolicy)
       self.default_scan_policy = create_policy(policies[:scan], ScanPolicy)
       self.default_write_policy = create_policy(policies[:write], WritePolicy)
+      self.default_operate_policy = create_policy(policies[:operate], OperatePolicy)
     end
 
-    def send_info_command(policy, command)
+    def send_info_command(policy, command, node = nil)
       Aerospike.logger.debug { "Sending info command: #{command}" }
-      _, response = @cluster.request_info(policy, command).first
+      if node
+        _, response = @cluster.request_node_info(node, policy, command).first
+      else
+        _, response = @cluster.request_info(policy, command).first
+      end
       response.to_s
     end
 
@@ -884,17 +901,16 @@ module Aerospike
       command.execute
     end
 
-    def execute_batch_index_commands(keys)
+    def execute_batch_index_commands(policy, keys)
       if @cluster.nodes.empty?
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing Batch Index command failed because cluster is empty.")
       end
 
-      batch_nodes = BatchIndexNode.generate_list(@cluster, keys)
+      batch_nodes = BatchIndexNode.generate_list(@cluster, policy.replica, keys)
       threads = []
 
       batch_nodes.each do |batch|
         threads << Thread.new do
-          Thread.current.abort_on_exception = true
           command = yield batch.node, batch
           execute_command(command)
         end
@@ -903,12 +919,12 @@ module Aerospike
       threads.each(&:join)
     end
 
-    def execute_batch_direct_commands(keys)
+    def execute_batch_direct_commands(policy, keys)
       if @cluster.nodes.empty?
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing Batch Direct command failed because cluster is empty.")
       end
 
-      batch_nodes = BatchDirectNode.generate_list(@cluster, keys)
+      batch_nodes = BatchDirectNode.generate_list(@cluster, policy.replica, keys)
       threads = []
 
       # Use a thread per namespace per node

data/lib/aerospike/cluster.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -28,6 +28,7 @@ module Aerospike
     attr_reader :features, :tls_options
     attr_reader :cluster_id, :aliases
     attr_reader :cluster_name
+    attr_accessor :rack_aware, :rack_id
 
     def initialize(policy, hosts)
       @cluster_seeds = hosts
@@ -37,6 +38,10 @@ module Aerospike
       @tend_interval = policy.tend_interval
       @cluster_name = policy.cluster_name
       @tls_options = policy.tls
+      @rack_aware = policy.rack_aware
+      @rack_id = policy.rack_id
+
+      @replica_index = Atomic.new(0)
 
       @aliases = {}
       @cluster_nodes = []
@@ -102,18 +107,128 @@ module Aerospike
       (node_array.length > 0) && !@closed.value
     end
 
-    def get_node_for_key(key)
-      partition = Partition.new_by_key(key)
+    # Returns a node on the cluster for read operations
+    def batch_read_node(partition, replica_policy)
+      case replica_policy
+        when Aerospike::Replica::MASTER, Aerospike::Replica::SEQUENCE
+          return master_node(partition)
+        when Aerospike::Replica::MASTER_PROLES
+          return master_proles_node(partition)
+        when Aerospike::Replica::PREFER_RACK
+          return rack_node(partition, seq)
+        when Aerospike::Replica::RANDOM
+          return random_node
+        else
+          raise Aerospike::Exceptions::InvalidNode("invalid policy.replica value")
+      end
+    end
+
+    # Returns a node on the cluster for read operations
+    def read_node(partition, replica_policy, seq)
+      case replica_policy
+        when Aerospike::Replica::MASTER
+          return master_node(partition)
+        when Aerospike::Replica::MASTER_PROLES
+          return master_proles_node(partition)
+        when Aerospike::Replica::PREFER_RACK
+          return rack_node(partition, seq)
+        when Aerospike::Replica::SEQUENCE
+          return sequence_node(partition, seq)
+        when Aerospike::Replica::RANDOM
+          return random_node
+        else
+          raise Aerospike::Exceptions::InvalidNode("invalid policy.replica value")
+      end
+    end
+
+    # Returns a node on the cluster for read operations
+    def master_node(partition)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      node_array = (replica_array.get)[0]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !node_array
+
+      node = (node_array.get)[partition.partition_id]
+      raise Aerospike::Exceptions::InvalidNode if !node || !node.active?
+
+      node
+    end
+
+    # Returns a node on the cluster
+    def rack_node(partition, seq)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      is_retry = seq.value > -1
+
+      node = nil
+      fallback = nil
+      for i in 1..replica_array.length
+        idx = (seq.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
+
+        next if !node
+
+        fallback = node
+
+        # If fallback exists, do not retry on node where command failed,
+        # even if fallback is not on the same rack.
+        return fallback if is_retry && fallback && i == replica_array.length
+
+        return node if node && node.active? && node.has_rack(partition.namespace, @rack_id)
+      end
+
+      return fallback if fallback
+
+      raise Aerospike::Exceptions::InvalidNode
+    end
 
-      # Must copy hashmap reference for copy on write semantics to work.
-      nmap = partitions
-      if node_array = nmap[partition.namespace]
-        node = node_array.value[partition.partition_id]
+    # Returns a node on the cluster for read operations
+    def master_proles_node(partition)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      node = nil
+      for replica in replica_array
+        idx = (@replica_index.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
+
+        return node if node && node.active?
+      end
+
+      raise Aerospike::Exceptions::InvalidNode
+    end
+
+    # Returns a random node on the cluster
+    def sequence_node(partition, seq)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      node = nil
+      for replica in replica_array
+        idx = (seq.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
 
         return node if node && node.active?
       end
 
-      random_node
+      raise Aerospike::Exceptions::InvalidNode
+    end
+
+    def get_node_for_key(replica_policy, key)
+      partition = Partition.new_by_key(key)
+      batch_read_node(partition, replica_policy)
     end
 
     # Returns a random node on the cluster
@@ -124,8 +239,8 @@ module Aerospike
       i = 0
       while i < length
         # Must handle concurrency with other non-tending threads, so node_index is consistent.
-        index = (@node_index.update{ |v| v+1 } % node_array.length).abs
-        node = node_array[index]
+        idx = (@node_index.update{ |v| v.succ } % node_array.length).abs
+        node = node_array[idx]
 
         return node if node.active?
 
@@ -167,12 +282,9 @@ module Aerospike
       end
     end
 
-    def update_partitions(tokens, node)
-      nmap = tokens.update_partition(partitions, node)
-      # update partition write map
+    def update_partitions(parser)
+      nmap = parser.update_partitions(partitions)
       set_partitions(nmap) if nmap
-
-      Aerospike.logger.info("Partitions for node #{node.name} updated")
     end
 
     def request_info(policy, *commands)
@@ -183,6 +295,13 @@ module Aerospike
       end
     end
 
+    def request_node_info(node, policy, *commands)
+      conn = node.get_connection(policy.timeout)
+      Info.request(conn, *commands).tap do
+        node.put_connection(conn)
+      end
+    end
+
     def supports_feature?(feature)
       @features.get.include?(feature.to_s)
     end
@@ -279,6 +398,7 @@ module Aerospike
 
       nodes.each do |node|
         node.refresh_partitions(peers) if node.partition_generation.changed?
+        node.refresh_racks if node.rebalance_generation.changed?
       end
 
       if peers.generation_changed? || !peers.use_peers?
@@ -441,8 +561,10 @@ module Aerospike
     def find_node_in_partition_map(filter)
       partitions_list = partitions
 
-      partitions_list.values.each do |node_array|
-        return true if node_array.value.any? { |node| node == filter }
+      partitions_list.values.each do |replica_array|
+        replica_array.get.each do |node_array|
+          return true if node_array.value.any? { |node| node == filter }
+        end
       end
       false
     end

data/lib/aerospike/cluster/partition.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cluster/partition_parser.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'base64'
+
+module Aerospike
+
+  class PartitionParser #:nodoc:
+
+    attr_accessor :copied, :partition_generation
+
+    PARTITION_GENERATION = "partition-generation";
+    REPLICAS_ALL = "replicas-all";
+
+    def initialize(node, conn)
+      @node = node
+      @conn = conn
+    end
+
+    def update_partitions(current_map)
+      # Use low-level info methods and parse byte array directly for maximum performance.
+      # Receive format: replicas-all\t
+      #                 <ns1>:<count>,<base 64 encoded bitmap1>,<base 64 encoded bitmap2>...;
+      #                 <ns2>:<count>,<base 64 encoded bitmap1>,<base 64 encoded bitmap2>...;\n
+      info_map = Info.request(@conn, PARTITION_GENERATION, REPLICAS_ALL)
+
+      @partition_generation = info_map[PARTITION_GENERATION].to_i
+
+      info = info_map[REPLICAS_ALL]
+      if !info || info.length == 0
+        raise Aerospike::Exceptions::Connection.new("#{REPLICAS_ALL} response for node #{@node.name} is empty")
+      end
+
+      @buffer = info
+      @length = info.length
+      @offset = 0
+
+      new_map = nil
+      copied = false
+      beginning = @offset
+
+      while @offset < @length && @buffer[@offset] != '\n'
+        namespace = parse_name
+        replica_count = parse_replica_count
+
+        replica_array = current_map[namespace]
+        if !replica_array
+          if !copied
+            # Make shallow copy of map.
+            new_map = current_map.clone
+            copied = true
+          end
+
+          replica_array = Atomic.new(Array.new(replica_count))
+          new_map[namespace] = replica_array
+        end
+
+        for replica in 0...replica_count do
+          node_array = (replica_array.get)[replica]
+
+          if !node_array
+            if !copied
+              # Make shallow copy of map.
+              new_map = current_map.clone
+              copied = true
+            end
+
+            node_array = Atomic.new(Array.new(Aerospike::Node::PARTITIONS))
+            new_map[namespace].update{|v| v[replica] = node_array; v}
+          end
+
+          restore_buffer = parse_bitmap
+          i = 0
+          while i < Aerospike::Node::PARTITIONS
+            if (restore_buffer[i>>3].ord & (0x80 >> (i & 7))) != 0
+              node_array.update{|v| v[i] = @node; v}
+            end
+            i = i.succ
+          end
+        end
+      end
+
+      copied ? new_map : nil
+    end
+
+    private
+
+    def parse_name
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ':'
+        @offset+=1
+      end
+
+      # Parse namespace.
+      namespace = @buffer[beginning...@offset].strip
+
+      if namespace.length <= 0 || namespace.length >= 32
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid partition namespace #{namespace}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      namespace
+    end
+
+    def parse_replica_count
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ','
+        @offset+=1
+      end
+
+      # Parse count
+      count = @buffer[beginning...@offset].strip.to_i
+
+      if count < 0 || count > 4096
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid partition count #{count}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      count
+    end
+
+    def parse_bitmap
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ','
+        break if @buffer[@offset] == ';'
+        @offset+=1
+      end
+
+      bit_map_length = @offset - beginning
+      restore_buffer = Base64.strict_decode64(@buffer[beginning, bit_map_length])
+
+      @offset+=1
+      restore_buffer
+    end
+
+
+    def get_truncated_response
+      max = @length
+      @length = max if @length > 200
+      @buffer[0...max]
+    end
+
+
+  end # class
+
+end # module