aerospike 2.12.0 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4509fc71b11809038c6ea0a8853e92de02fcade0f8e29723e7507a029f7a5e1
-  data.tar.gz: 8fd24ad2f984e728c9e6137b7bf6527aaa07a1371c9d46539de7f066048b13e0
+  metadata.gz: 7bef523a7e7500d8e39cb80f3b9ee6831a5ddc0bd6d782685181cae6a6c6ada5
+  data.tar.gz: 760d626bffac6ed69bcdda048daf39b07a12a85264b67be0677ad5d552c5473c
 SHA512:
-  metadata.gz: 41aaaabe27c7f9a9080fb33a10444bef59f5ba6482b775d6b812529b3cc4f4656665a4e7294d328df3048c18519fafd99ac448b333c7a0506c78305a685c579e
-  data.tar.gz: adf16fbcb8247ac8724d1823a1072cf5796f52477fb5a9bbe84f17a6ad71402f33b3ba2db9777a8b907737d291b7e5d9a98fa6ec7595093c5fb2d0cf24915de8
+  metadata.gz: df19a0c2900463a35b73ba52bfae3bdb2c5f6cbdaba3ab17172219e96ad3de3878bbc1ff57403fbbffe0fb48cc147ee642297e2fe47c829d768e76b1e4dee0fa
+  data.tar.gz: 528ade6f9abfb9b9e266439fefa6606f2623aec2503ec6c5466a5c3f9ffd9656debea49ff434f715f042c3ae319546e80352aa36cdc925ed351cab2633c75c4b

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
+## [2.13.0] - 2019-07-17
+
+* **New Features**
+  * Adds support for replica policies.
+
+* **Improvements**
+  * Remove support for "old" partition tokenizer.
+  * Refactor how partition parser is initialized and called.
+  * Adds support for 'replicas' and remove the old partition table queries from the server.
+
 ## [2.12.0] - 2019-04-21
 
 * **New Features**

data/lib/aerospike.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -101,11 +101,10 @@ require 'aerospike/connection/create'
 
 require 'aerospike/cluster'
 require 'aerospike/cluster/create_connection'
-require 'aerospike/cluster/partition'
 require 'aerospike/cluster/find_nodes_to_remove'
 require 'aerospike/cluster/find_node'
-require 'aerospike/cluster/partition_tokenizer_new'
-require 'aerospike/cluster/partition_tokenizer_old'
+require 'aerospike/cluster/partition'
+require 'aerospike/cluster/partition_parser'
 require 'aerospike/node'
 require 'aerospike/node/generation'
 require 'aerospike/node/refresh/failed'

data/lib/aerospike/aerospike_exception.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -90,5 +90,11 @@ module Aerospike
         super(ResultCode::COMMAND_REJECTED, msg)
       end
     end
+
+    class InvalidNamespace < Aerospike
+      def initialize(msg=nil)
+        super(ResultCode::INVALID_NAMESPACE, msg)
+      end
+    end
   end
 end

data/lib/aerospike/atomic/atomic.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/aerospike/bin.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/list_operation.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_operation.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_order.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2017 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_policy.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_return_type.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2017 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_write_mode.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/client.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -322,11 +322,11 @@ module Aerospike
 
       if policy.use_batch_direct
         key_map = BatchItem.generate_map(keys)
-        execute_batch_direct_commands(keys) do |node, batch|
+        execute_batch_direct_commands(policy, keys) do |node, batch|
           BatchDirectCommand.new(node, batch, policy, key_map, bin_names, results, info_flags)
         end
       else
-        execute_batch_index_commands(keys) do |node, batch|
+        execute_batch_index_commands(policy, keys) do |node, batch|
           BatchIndexCommand.new(node, batch, policy, bin_names, results, info_flags)
         end
       end
@@ -351,11 +351,11 @@ module Aerospike
 
       if policy.use_batch_direct
         key_map = BatchItem.generate_map(keys)
-        execute_batch_direct_commands(keys) do |node, batch|
+        execute_batch_direct_commands(policy, keys) do |node, batch|
           BatchDirectExistsCommand.new(node, batch, policy, key_map, results)
         end
       else
-        execute_batch_index_commands(keys) do |node, batch|
+        execute_batch_index_commands(policy, keys) do |node, batch|
           BatchIndexExistsCommand.new(node, batch, policy, results)
         end
       end
@@ -884,12 +884,12 @@ module Aerospike
       command.execute
     end
 
-    def execute_batch_index_commands(keys)
+    def execute_batch_index_commands(policy, keys)
       if @cluster.nodes.empty?
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing Batch Index command failed because cluster is empty.")
       end
 
-      batch_nodes = BatchIndexNode.generate_list(@cluster, keys)
+      batch_nodes = BatchIndexNode.generate_list(@cluster, policy.replica, keys)
       threads = []
 
       batch_nodes.each do |batch|
@@ -902,12 +902,12 @@ module Aerospike
       threads.each(&:join)
     end
 
-    def execute_batch_direct_commands(keys)
+    def execute_batch_direct_commands(policy, keys)
       if @cluster.nodes.empty?
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing Batch Direct command failed because cluster is empty.")
       end
 
-      batch_nodes = BatchDirectNode.generate_list(@cluster, keys)
+      batch_nodes = BatchDirectNode.generate_list(@cluster, policy.replica, keys)
       threads = []
 
       # Use a thread per namespace per node

data/lib/aerospike/cluster.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -38,6 +38,8 @@ module Aerospike
       @cluster_name = policy.cluster_name
       @tls_options = policy.tls
 
+      @replica_index = Atomic.new(0)
+
       @aliases = {}
       @cluster_nodes = []
       @partition_write_map = {}
@@ -102,18 +104,92 @@ module Aerospike
       (node_array.length > 0) && !@closed.value
     end
 
-    def get_node_for_key(key)
-      partition = Partition.new_by_key(key)
+    # Returns a node on the cluster for read operations
+    def batch_read_node(partition, replica_policy)
+      case replica_policy
+        when Aerospike::Replica::MASTER, Aerospike::Replica::SEQUENCE
+          return master_node(partition)
+        when Aerospike::Replica::MASTER_PROLES
+          return master_proles_node(partition)
+        when Aerospike::Replica::RANDOM
+          return random_node
+        else
+          raise Aerospike::Exceptions::InvalidNode("invalid policy.replica value")
+      end
+    end
+
+    # Returns a node on the cluster for read operations
+    def read_node(partition, replica_policy, seq)
+      case replica_policy
+        when Aerospike::Replica::MASTER
+          return master_node(partition)
+        when Aerospike::Replica::MASTER_PROLES
+          return master_proles_node(partition)
+        when Aerospike::Replica::SEQUENCE
+          return sequence_node(partition, seq)
+        when Aerospike::Replica::RANDOM
+          return random_node
+        else
+          raise Aerospike::Exceptions::InvalidNode("invalid policy.replica value")
+      end
+    end
+
+    # Returns a node on the cluster for read operations
+    def master_node(partition)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      node_array = (replica_array.get)[0]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !node_array
+
+      node = (node_array.get)[partition.partition_id]
+      raise Aerospike::Exceptions::InvalidNode if !node || !node.active?
+
+      node
+    end
+
+    # Returns a node on the cluster for read operations
+    def master_proles_node(partition)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
 
-      # Must copy hashmap reference for copy on write semantics to work.
-      nmap = partitions
-      if node_array = nmap[partition.namespace]
-        node = node_array.value[partition.partition_id]
+      node = nil
+      for replica in replica_array
+        idx = (@replica_index.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
 
         return node if node && node.active?
       end
 
-      random_node
+      raise Aerospike::Exceptions::InvalidNode
+    end
+
+    # Returns a random node on the cluster
+    def sequence_node(partition, seq)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      node = nil
+      for replica in replica_array
+        idx = (seq.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
+
+        return node if node && node.active?
+      end
+
+      raise Aerospike::Exceptions::InvalidNode
+    end
+
+    def get_node_for_key(replica_policy, key)
+      partition = Partition.new_by_key(key)
+      batch_read_node(partition, replica_policy)
     end
 
     # Returns a random node on the cluster
@@ -124,8 +200,8 @@ module Aerospike
       i = 0
       while i < length
         # Must handle concurrency with other non-tending threads, so node_index is consistent.
-        index = (@node_index.update{ |v| v+1 } % node_array.length).abs
-        node = node_array[index]
+        idx = (@node_index.update{ |v| v.succ } % node_array.length).abs
+        node = node_array[idx]
 
         return node if node.active?
 
@@ -167,12 +243,9 @@ module Aerospike
       end
     end
 
-    def update_partitions(tokens, node)
-      nmap = tokens.update_partition(partitions, node)
-      # update partition write map
+    def update_partitions(parser)
+      nmap = parser.update_partitions(partitions)
       set_partitions(nmap) if nmap
-
-      Aerospike.logger.info("Partitions for node #{node.name} updated")
     end
 
     def request_info(policy, *commands)
@@ -441,8 +514,10 @@ module Aerospike
     def find_node_in_partition_map(filter)
       partitions_list = partitions
 
-      partitions_list.values.each do |node_array|
-        return true if node_array.value.any? { |node| node == filter }
+      partitions_list.values.each do |replica_array|
+        replica_array.get.each do |node_array|
+          return true if node_array.value.any? { |node| node == filter }
+        end
       end
       false
     end

data/lib/aerospike/cluster/partition.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cluster/partition_parser.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'base64'
+
+module Aerospike
+
+  class PartitionParser #:nodoc:
+
+    attr_accessor :copied, :partition_generation
+
+    PARTITION_GENERATION = "partition-generation";
+    REPLICAS_ALL = "replicas-all";
+
+    def initialize(node, conn)
+      @node = node
+      @conn = conn
+    end
+
+    def update_partitions(current_map)
+      # Use low-level info methods and parse byte array directly for maximum performance.
+      # Receive format: replicas-all\t
+      #                 <ns1>:<count>,<base 64 encoded bitmap1>,<base 64 encoded bitmap2>...;
+      #                 <ns2>:<count>,<base 64 encoded bitmap1>,<base 64 encoded bitmap2>...;\n
+      info_map = Info.request(@conn, PARTITION_GENERATION, REPLICAS_ALL)
+
+      @partition_generation = info_map[PARTITION_GENERATION].to_i
+
+      info = info_map[REPLICAS_ALL]
+      if !info || info.length == 0
+        raise Aerospike::Exceptions::Connection.new("#{REPLICAS_ALL} response for node #{@node.name} is empty")
+      end
+
+      @buffer = info
+      @length = info.length
+      @offset = 0
+
+      new_map = nil
+      copied = false
+      beginning = @offset
+
+      while @offset < @length && @buffer[@offset] != '\n'
+        namespace = parse_name
+        replica_count = parse_replica_count
+
+        replica_array = current_map[namespace]
+        if !replica_array
+          if !copied
+            # Make shallow copy of map.
+            new_map = current_map.clone
+            copied = true
+          end
+
+          replica_array = Atomic.new(Array.new(replica_count))
+          new_map[namespace] = replica_array
+        end
+
+        for replica in 0...replica_count do
+          node_array = (replica_array.get)[replica]
+
+          if !node_array
+            if !copied
+              # Make shallow copy of map.
+              new_map = current_map.clone
+              copied = true
+            end
+
+            node_array = Atomic.new(Array.new(Aerospike::Node::PARTITIONS))
+            new_map[namespace].update{|v| v[replica] = node_array; v}
+          end
+
+          restore_buffer = parse_bitmap
+          i = 0
+          while i < Aerospike::Node::PARTITIONS
+            if (restore_buffer[i>>3].ord & (0x80 >> (i & 7))) != 0
+              node_array.update{|v| v[i] = @node; v}
+            end
+            i = i.succ
+          end
+        end
+      end
+
+      copied ? new_map : nil
+    end
+
+    private
+
+    def parse_name
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ':'
+        @offset+=1
+      end
+
+      # Parse namespace.
+      namespace = @buffer[beginning...@offset].strip
+
+      if namespace.length <= 0 || namespace.length >= 32
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid partition namespace #{namespace}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      namespace
+    end
+
+    def parse_replica_count
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ','
+        @offset+=1
+      end
+
+      # Parse count
+      count = @buffer[beginning...@offset].strip.to_i
+
+      if count < 0 || count > 4096
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid partition count #{count}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      count
+    end
+
+    def parse_bitmap
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ','
+        break if @buffer[@offset] == ';'
+        @offset+=1
+      end
+
+      bit_map_length = @offset - beginning
+      restore_buffer = Base64.strict_decode64(@buffer[beginning, bit_map_length])
+
+      @offset+=1
+      restore_buffer
+    end
+
+
+    def get_truncated_response
+      max = @length
+      @length = max if @length > 200
+      @buffer[0...max]
+    end
+
+
+  end # class
+
+end # module