aerospike 2.9.0 → 2.13.0

data/lib/aerospike/cluster/partition.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cluster/partition_parser.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'base64'
+
+module Aerospike
+
+  class PartitionParser #:nodoc:
+
+    attr_accessor :copied, :partition_generation
+
+    PARTITION_GENERATION = "partition-generation";
+    REPLICAS_ALL = "replicas-all";
+
+    def initialize(node, conn)
+      @node = node
+      @conn = conn
+    end
+
+    def update_partitions(current_map)
+      # Use low-level info methods and parse byte array directly for maximum performance.
+      # Receive format: replicas-all\t
+      #                 <ns1>:<count>,<base 64 encoded bitmap1>,<base 64 encoded bitmap2>...;
+      #                 <ns2>:<count>,<base 64 encoded bitmap1>,<base 64 encoded bitmap2>...;\n
+      info_map = Info.request(@conn, PARTITION_GENERATION, REPLICAS_ALL)
+
+      @partition_generation = info_map[PARTITION_GENERATION].to_i
+
+      info = info_map[REPLICAS_ALL]
+      if !info || info.length == 0
+        raise Aerospike::Exceptions::Connection.new("#{REPLICAS_ALL} response for node #{@node.name} is empty")
+      end
+
+      @buffer = info
+      @length = info.length
+      @offset = 0
+
+      new_map = nil
+      copied = false
+      beginning = @offset
+
+      while @offset < @length && @buffer[@offset] != '\n'
+        namespace = parse_name
+        replica_count = parse_replica_count
+
+        replica_array = current_map[namespace]
+        if !replica_array
+          if !copied
+            # Make shallow copy of map.
+            new_map = current_map.clone
+            copied = true
+          end
+
+          replica_array = Atomic.new(Array.new(replica_count))
+          new_map[namespace] = replica_array
+        end
+
+        for replica in 0...replica_count do
+          node_array = (replica_array.get)[replica]
+
+          if !node_array
+            if !copied
+              # Make shallow copy of map.
+              new_map = current_map.clone
+              copied = true
+            end
+
+            node_array = Atomic.new(Array.new(Aerospike::Node::PARTITIONS))
+            new_map[namespace].update{|v| v[replica] = node_array; v}
+          end
+
+          restore_buffer = parse_bitmap
+          i = 0
+          while i < Aerospike::Node::PARTITIONS
+            if (restore_buffer[i>>3].ord & (0x80 >> (i & 7))) != 0
+              node_array.update{|v| v[i] = @node; v}
+            end
+            i = i.succ
+          end
+        end
+      end
+
+      copied ? new_map : nil
+    end
+
+    private
+
+    def parse_name
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ':'
+        @offset+=1
+      end
+
+      # Parse namespace.
+      namespace = @buffer[beginning...@offset].strip
+
+      if namespace.length <= 0 || namespace.length >= 32
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid partition namespace #{namespace}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      namespace
+    end
+
+    def parse_replica_count
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ','
+        @offset+=1
+      end
+
+      # Parse count
+      count = @buffer[beginning...@offset].strip.to_i
+
+      if count < 0 || count > 4096
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid partition count #{count}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      count
+    end
+
+    def parse_bitmap
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ','
+        break if @buffer[@offset] == ';'
+        @offset+=1
+      end
+
+      bit_map_length = @offset - beginning
+      restore_buffer = Base64.strict_decode64(@buffer[beginning, bit_map_length])
+
+      @offset+=1
+      restore_buffer
+    end
+
+
+    def get_truncated_response
+      max = @length
+      @length = max if @length > 200
+      @buffer[0...max]
+    end
+
+
+  end # class
+
+end # module

data/lib/aerospike/command/admin_command.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -25,7 +25,6 @@ module Aerospike
   CHANGE_PASSWORD = 4
   GRANT_ROLES     = 5
   REVOKE_ROLES    = 6
-  REPLACE_ROLES   = 7
   #CREATE_ROLE = 8
   QUERY_USERS = 9
   #QUERY_ROLES =  10
@@ -39,7 +38,7 @@ module Aerospike
   #PRIVILEGES =  11
 
   # Misc
-  MSG_VERSION  = 0
+  MSG_VERSION  = 2
   MSG_TYPE     = 2
 
   HEADER_SIZE       = 24

data/lib/aerospike/command/batch_direct_command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/batch_direct_exists_command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/batch_direct_node.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -24,8 +24,8 @@ module Aerospike
     attr_accessor :node
     attr_accessor :batch_namespaces
 
-    def self.generate_list(cluster, keys)
-      keys.group_by { |key| cluster.get_node_for_key(key) }
+    def self.generate_list(cluster, replica_policy, keys)
+      keys.group_by { |key| cluster.get_node_for_key(replica_policy, key) }
         .map { |node, keys_for_node| BatchDirectNode.new(node, keys_for_node) }
     end
 

data/lib/aerospike/command/batch_index_command.rb

@@ -39,7 +39,12 @@ module Aerospike
     def write_buffer
       bin_name_size = 0
       operation_count = 0
+      field_count_row = 1
       field_count = 1
+
+      predexp_size = estimate_predexp(@policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       if bin_names
         bin_names.each do |bin_name|
           bin_name_size += bin_name.bytesize + OPERATION_HEADER_SIZE
@@ -61,7 +66,10 @@ module Aerospike
         end
       end
       size_buffer
-      write_header(policy,read_attr | INFO1_BATCH, 0, 1, 0)
+      write_header(policy,read_attr | INFO1_BATCH, 0, field_count, 0)
+
+      write_predexp(@policy.predexp, predexp_size)
+
       write_field_header(0, Aerospike::FieldType::BATCH_INDEX)
       @data_offset += @data_buffer.write_int32(batch.keys.length, @data_offset)
       @data_offset += @data_buffer.write_byte(1, @data_offset)
@@ -77,7 +85,7 @@ module Aerospike
         else
           @data_offset += @data_buffer.write_byte(0, @data_offset)
           @data_offset += @data_buffer.write_byte(read_attr, @data_offset)
-          @data_offset += @data_buffer.write_int16(field_count, @data_offset)
+          @data_offset += @data_buffer.write_int16(field_count_row, @data_offset)
           @data_offset += @data_buffer.write_int16(operation_count, @data_offset)
           write_field_string(key.namespace, Aerospike::FieldType::NAMESPACE)
 

data/lib/aerospike/command/batch_index_node.rb

@@ -22,9 +22,9 @@ module Aerospike
     attr_accessor :node
     attr_accessor :keys_by_idx
 
-    def self.generate_list(cluster, keys)
+    def self.generate_list(cluster, replica_policy, keys)
       keys.each_with_index
-        .group_by { |key, _| cluster.get_node_for_key(key) }
+        .group_by { |key, _| cluster.get_node_for_key(replica_policy, key) }
         .map { |node, keys_with_idx| BatchIndexNode.new(node, keys_with_idx) }
     end
 

data/lib/aerospike/command/batch_item.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -77,14 +77,20 @@ module Aerospike
 
   class Command #:nodoc:
 
-    def initialize(node)
+    def initialize(node=nil)
+      @data_offset = 0
+      @data_buffer = nil
+
       @node = node
 
+      # will add before use
+      @sequence = Atomic.new(-1)
+
       self
     end
 
     # List of all bins that this command will write to - sub-classes should
-    # overrite this as appropriate.
+    # override this as appropriate.
     def write_bins
       []
     end
@@ -93,6 +99,9 @@ module Aerospike
     def set_write(policy, operation, key, bins)
       begin_cmd
       field_count = estimate_key_size(key, policy)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
 
       bins.each do |bin|
         estimate_operation_size_for_bin(bin)
@@ -102,6 +111,7 @@ module Aerospike
 
       write_header_with_policy(policy, 0, INFO2_WRITE, field_count, bins.length)
       write_key(key, policy)
+      write_predexp(policy.predexp, predexp_size)
 
       bins.each do |bin|
         write_operation_for_bin(bin, operation)
@@ -114,9 +124,14 @@ module Aerospike
     def set_delete(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       size_buffer
       write_header_with_policy(policy, 0, INFO2_WRITE|INFO2_DELETE, field_count, 0)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       end_cmd
     end
 
@@ -124,10 +139,15 @@ module Aerospike
     def set_touch(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       estimate_operation_size
       size_buffer
       write_header_with_policy(policy, 0, INFO2_WRITE, field_count, 1)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       write_operation_for_operation_type(Aerospike::Operation::TOUCH)
       end_cmd
     end
@@ -136,9 +156,14 @@ module Aerospike
     def set_exists(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       size_buffer
       write_header(policy, INFO1_READ|INFO1_NOBINDATA, 0, field_count, 0)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       end_cmd
     end
 
@@ -146,9 +171,14 @@ module Aerospike
     def set_read_for_key_only(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       size_buffer
       write_header(policy, INFO1_READ|INFO1_GET_ALL, 0, field_count, 0)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       end_cmd
     end
 
@@ -157,6 +187,10 @@ module Aerospike
       if bin_names && bin_names.length > 0
         begin_cmd
         field_count = estimate_key_size(key)
+        
+        predexp_size = estimate_predexp(policy.predexp)
+        field_count += 1 if predexp_size > 0
+
 
         bin_names.each do |bin_name|
           estimate_operation_size_for_bin_name(bin_name)
@@ -165,6 +199,7 @@ module Aerospike
         size_buffer
         write_header(policy, INFO1_READ, 0, field_count, bin_names.length)
         write_key(key)
+        write_predexp(policy.predexp, predexp_size)
 
         bin_names.each do |bin_name|
           write_operation_for_bin_name(bin_name, Aerospike::Operation::READ)
@@ -180,6 +215,10 @@ module Aerospike
     def set_read_header(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       estimate_operation_size_for_bin_name('')
       size_buffer
 
@@ -190,6 +229,7 @@ module Aerospike
       write_header(policy, INFO1_READ, 0, field_count, 1)
 
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       write_operation_for_bin_name('', Aerospike::Operation::READ)
       end_cmd
     end
@@ -198,6 +238,10 @@ module Aerospike
     def set_operate(policy, key, operations)
       begin_cmd
       field_count = estimate_key_size(key, policy)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       read_attr = 0
       write_attr = 0
       read_header = false
@@ -205,19 +249,22 @@ module Aerospike
       operations.each do |operation|
         case operation.op_type
         when Aerospike::Operation::READ
-            read_attr |= INFO1_READ
+          read_attr |= INFO1_READ
 
           # Read all bins if no bin is specified.
           read_attr |= INFO1_GET_ALL unless operation.bin_name
 
         when Aerospike::Operation::READ_HEADER
-            # The server does not currently return record header data with _INFO1_NOBINDATA attribute set.
-            # The workaround is to request a non-existent bin.
-            # TODO: Fix this on server.
-            # read_attr |= _INFO1_READ | _INFO1_NOBINDATA
-            read_attr |= INFO1_READ
+          # The server does not currently return record header data with _INFO1_NOBINDATA attribute set.
+          # The workaround is to request a non-existent bin.
+          # TODO: Fix this on server.
+          # read_attr |= _INFO1_READ | _INFO1_NOBINDATA
+          read_attr |= INFO1_READ
           read_header = true
 
+        when Aerospike::Operation::CDT_READ
+          read_attr |= INFO1_READ
+
         else
           write_attr = INFO2_WRITE
         end
@@ -232,6 +279,7 @@ module Aerospike
         write_header(policy, read_attr, write_attr, field_count, operations.length)
       end
       write_key(key, policy)
+      write_predexp(policy.predexp, predexp_size)
 
       operations.each do |operation|
         write_operation_for_operation(operation)
@@ -245,6 +293,10 @@ module Aerospike
     def set_udf(policy, key, package_name, function_name, args)
       begin_cmd
       field_count = estimate_key_size(key, policy)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       arg_bytes = args.to_bytes
 
       field_count += estimate_udf_size(package_name, function_name, arg_bytes)
@@ -252,6 +304,7 @@ module Aerospike
 
       write_header(policy, 0, INFO2_WRITE, field_count, 0)
       write_key(key, policy)
+      write_predexp(policy.predexp, predexp_size)
       write_field_string(package_name, Aerospike::FieldType::UDF_PACKAGE_NAME)
       write_field_string(function_name, Aerospike::FieldType::UDF_FUNCTION)
       write_field_bytes(arg_bytes, Aerospike::FieldType::UDF_ARGLIST)
@@ -273,6 +326,14 @@ module Aerospike
         @data_offset += set_name.bytesize + FIELD_HEADER_SIZE
         field_count += 1
       end
+      
+      if policy.records_per_second > 0
+        @data_offset += 4 + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
 
       # Estimate scan options size.
       @data_offset += 2 + FIELD_HEADER_SIZE
@@ -310,6 +371,12 @@ module Aerospike
         write_field_string(set_name, Aerospike::FieldType::TABLE)
       end
 
+      if policy.records_per_second > 0
+        write_field_int(policy.records_per_second, Aerospike::FieldType::RECORDS_PER_SECOND)
+      end
+
+      write_predexp(policy.predexp, predexp_size)
+
       write_field_header(2, Aerospike::FieldType::SCAN_OPTIONS)
 
       priority = policy.priority & 0xFF
@@ -355,6 +422,7 @@ module Aerospike
         break if @policy.timeout > 0 && Time.now > limit
 
         begin
+          @node = get_node
           @conn = @node.get_connection(@policy.timeout)
         rescue => e
           # Socket connection error has occurred. Decrease health and retry.
@@ -498,6 +566,16 @@ module Aerospike
       @data_offset += OPERATION_HEADER_SIZE
     end
 
+    def estimate_predexp(predexp)
+      if predexp && predexp.size > 0
+        @data_offset += FIELD_HEADER_SIZE
+        sz = Aerospike::PredExp.estimate_size(predexp)
+        @data_offset += sz
+        return sz
+      end
+      return 0
+    end
+
     # Generic header write.
     def write_header(policy, read_attr, write_attr, field_count, operation_count)
       read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
@@ -681,21 +759,34 @@ module Aerospike
       @data_offset += len
     end
 
+    def write_field_int(i, ftype)
+      @data_buffer.write_int32(i, @data_offset+FIELD_HEADER_SIZE)
+      write_field_header(4, ftype)
+      @data_offset += 4
+    end
+
     def write_field_bytes(bytes, ftype)
       @data_buffer.write_binary(bytes, @data_offset+FIELD_HEADER_SIZE)
-
       write_field_header(bytes.bytesize, ftype)
       @data_offset += bytes.bytesize
     end
 
     def write_field_header(size, ftype)
-      # Buffer.Int32ToBytes(size+1), @data_buffer, @data_offset
       @data_buffer.write_int32(size+1, @data_offset)
       @data_offset += 4
       @data_buffer.write_byte(ftype, @data_offset)
       @data_offset += 1
     end
 
+    def write_predexp(predexp, predexp_size)
+      if predexp && predexp.size > 0
+        write_field_header(predexp_size, Aerospike::FieldType::PREDEXP)
+        @data_offset = Aerospike::PredExp.write(
+          predexp, @data_buffer, @data_offset
+        )
+      end
+    end
+
     def begin_cmd
       @data_offset = MSG_TOTAL_HEADER_SIZE
     end