aerospike 2.11.0 → 2.16.0

data/lib/aerospike/cluster/rack_parser.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'base64'
+
+module Aerospike
+
+  class RackParser #:nodoc:
+
+    attr_accessor :rebalance_generation, :racks
+
+    REBALANCE_GENERATION = "rebalance-generation"
+    RACK_IDS = "rack-ids"
+
+    def initialize(node, conn)
+      @node = node
+      @conn = conn
+      @racks = nil
+    end
+
+    def update_racks
+      # Use low-level info methods and parse byte array directly for maximum performance.
+      # Receive format: rack-ids\t
+      #                 <ns1>:<rack-id>...;
+      #                 <ns2>:<rack-id>...; ...
+      info_map = Info.request(@conn, REBALANCE_GENERATION, RACK_IDS)
+
+      @rebalance_generation = info_map[REBALANCE_GENERATION].to_i
+
+      info = info_map[RACK_IDS]
+      if !info || info.length == 0
+        raise Aerospike::Exceptions::Connection.new("#{RACK_IDS} response for node #{@node.name} is empty")
+      end
+
+      @buffer = info
+      @length = info.length
+      @offset = 0
+
+      while @offset < @length && @buffer[@offset] != '\n'
+        namespace = parse_name
+        rack_id = parse_rack_id
+
+        @racks = {} if !@racks
+        @racks[namespace] = rack_id
+      end
+
+      @racks
+    end
+
+    private
+
+    def parse_name
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ':'
+        @offset+=1
+      end
+
+      # Parse namespace.
+      namespace = @buffer[beginning...@offset].strip
+
+      if namespace.length <= 0 || namespace.length >= 32
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid rack namespace #{namespace}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      namespace
+    end
+
+    def parse_rack_id
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ';'
+        @offset+=1
+      end
+
+      # Parse rack_id
+      rack_id = @buffer[beginning...@offset].strip.to_i
+
+      if rack_id < 0
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid rack_id #{rack_id}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      rack_id
+    end
+
+    def get_truncated_response
+      max = @length
+      @length = max if @length > 200
+      @buffer[0...max]
+    end
+
+
+  end # class
+
+end # module

data/lib/aerospike/command/admin_command.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/batch_direct_command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -75,6 +75,7 @@ module Aerospike
       end
 
       end_cmd
+      mark_compressed(@policy)
     end
 
     # Parse all results in the batch.  Add records to shared list.

data/lib/aerospike/command/batch_direct_exists_command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/batch_direct_node.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -24,8 +24,8 @@ module Aerospike
     attr_accessor :node
     attr_accessor :batch_namespaces
 
-    def self.generate_list(cluster, keys)
-      keys.group_by { |key| cluster.get_node_for_key(key) }
+    def self.generate_list(cluster, replica_policy, keys)
+      keys.group_by { |key| cluster.get_node_for_key(replica_policy, key) }
         .map { |node, keys_for_node| BatchDirectNode.new(node, keys_for_node) }
     end
 

data/lib/aerospike/command/batch_index_command.rb

@@ -39,7 +39,12 @@ module Aerospike
     def write_buffer
       bin_name_size = 0
       operation_count = 0
+      field_count_row = 1
       field_count = 1
+
+      predexp_size = estimate_predexp(@policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       if bin_names
         bin_names.each do |bin_name|
           bin_name_size += bin_name.bytesize + OPERATION_HEADER_SIZE
@@ -61,7 +66,10 @@ module Aerospike
         end
       end
       size_buffer
-      write_header(policy,read_attr | INFO1_BATCH, 0, 1, 0)
+      write_header(policy,read_attr | INFO1_BATCH, 0, field_count, 0)
+
+      write_predexp(@policy.predexp, predexp_size)
+
       write_field_header(0, Aerospike::FieldType::BATCH_INDEX)
       @data_offset += @data_buffer.write_int32(batch.keys.length, @data_offset)
       @data_offset += @data_buffer.write_byte(1, @data_offset)
@@ -77,7 +85,7 @@ module Aerospike
         else
           @data_offset += @data_buffer.write_byte(0, @data_offset)
           @data_offset += @data_buffer.write_byte(read_attr, @data_offset)
-          @data_offset += @data_buffer.write_int16(field_count, @data_offset)
+          @data_offset += @data_buffer.write_int16(field_count_row, @data_offset)
           @data_offset += @data_buffer.write_int16(operation_count, @data_offset)
           write_field_string(key.namespace, Aerospike::FieldType::NAMESPACE)
 
@@ -90,6 +98,7 @@ module Aerospike
         end
       end
       end_cmd
+      mark_compressed(@policy)
     end
 
     # Parse all results in the batch.  Add records to shared list.

data/lib/aerospike/command/batch_index_node.rb

@@ -22,9 +22,9 @@ module Aerospike
     attr_accessor :node
     attr_accessor :keys_by_idx
 
-    def self.generate_list(cluster, keys)
+    def self.generate_list(cluster, replica_policy, keys)
       keys.each_with_index
-        .group_by { |key, _| cluster.get_node_for_key(key) }
+        .group_by { |key, _| cluster.get_node_for_key(replica_policy, key) }
         .map { |node, keys_with_idx| BatchIndexNode.new(node, keys_with_idx) }
     end
 

data/lib/aerospike/command/batch_item.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -16,6 +16,7 @@
 # the License.
 
 require 'time'
+require 'zlib'
 
 require 'msgpack'
 require 'aerospike/result_code'
@@ -42,6 +43,9 @@ module Aerospike
   # Involve all replicas in read operation.
   INFO1_CONSISTENCY_ALL = Integer(1 << 6)
 
+  # Tell server to compress it's response.
+  INFO1_COMPRESS_RESPONSE = (1 << 7)
+
   # Create or update record
   INFO2_WRITE = Integer(1 << 0)
   # Fling a record into the belly of Moloch.
@@ -55,6 +59,9 @@ module Aerospike
   # Create only. Fail if record already exists.
   INFO2_CREATE_ONLY = Integer(1 << 5)
 
+  # Return a result for every operation.
+  INFO2_RESPOND_ALL_OPS = Integer(1 << 7)
+
   # This is the last of a multi-part message.
   INFO3_LAST = Integer(1 << 0)
   # Commit to master only before declaring success.
@@ -72,19 +79,29 @@ module Aerospike
   OPERATION_HEADER_SIZE      = 8
   MSG_REMAINING_HEADER_SIZE  = 22
   DIGEST_SIZE                = 20
+  COMPRESS_THRESHOLD         = 128
   CL_MSG_VERSION             = 2
   AS_MSG_TYPE                = 3
+  AS_MSG_TYPE_COMPRESSED     = 4
 
   class Command #:nodoc:
 
-    def initialize(node)
+    def initialize(node=nil)
+      @data_offset = 0
+      @data_buffer = nil
+
       @node = node
 
+      @compress = false
+
+      # will add before use
+      @sequence = Atomic.new(-1)
+
       self
     end
 
     # List of all bins that this command will write to - sub-classes should
-    # overrite this as appropriate.
+    # override this as appropriate.
     def write_bins
       []
     end
@@ -93,6 +110,9 @@ module Aerospike
     def set_write(policy, operation, key, bins)
       begin_cmd
       field_count = estimate_key_size(key, policy)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
 
       bins.each do |bin|
         estimate_operation_size_for_bin(bin)
@@ -102,21 +122,28 @@ module Aerospike
 
       write_header_with_policy(policy, 0, INFO2_WRITE, field_count, bins.length)
       write_key(key, policy)
+      write_predexp(policy.predexp, predexp_size)
 
       bins.each do |bin|
         write_operation_for_bin(bin, operation)
       end
 
       end_cmd
+      mark_compressed(policy)
     end
 
     # Writes the command for delete operations
     def set_delete(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       size_buffer
       write_header_with_policy(policy, 0, INFO2_WRITE|INFO2_DELETE, field_count, 0)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       end_cmd
     end
 
@@ -124,10 +151,15 @@ module Aerospike
     def set_touch(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       estimate_operation_size
       size_buffer
       write_header_with_policy(policy, 0, INFO2_WRITE, field_count, 1)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       write_operation_for_operation_type(Aerospike::Operation::TOUCH)
       end_cmd
     end
@@ -136,9 +168,14 @@ module Aerospike
     def set_exists(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       size_buffer
       write_header(policy, INFO1_READ|INFO1_NOBINDATA, 0, field_count, 0)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       end_cmd
     end
 
@@ -146,9 +183,14 @@ module Aerospike
     def set_read_for_key_only(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       size_buffer
       write_header(policy, INFO1_READ|INFO1_GET_ALL, 0, field_count, 0)
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       end_cmd
     end
 
@@ -157,6 +199,10 @@ module Aerospike
       if bin_names && bin_names.length > 0
         begin_cmd
         field_count = estimate_key_size(key)
+        
+        predexp_size = estimate_predexp(policy.predexp)
+        field_count += 1 if predexp_size > 0
+
 
         bin_names.each do |bin_name|
           estimate_operation_size_for_bin_name(bin_name)
@@ -165,6 +211,7 @@ module Aerospike
         size_buffer
         write_header(policy, INFO1_READ, 0, field_count, bin_names.length)
         write_key(key)
+        write_predexp(policy.predexp, predexp_size)
 
         bin_names.each do |bin_name|
           write_operation_for_bin_name(bin_name, Aerospike::Operation::READ)
@@ -180,6 +227,10 @@ module Aerospike
     def set_read_header(policy, key)
       begin_cmd
       field_count = estimate_key_size(key)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       estimate_operation_size_for_bin_name('')
       size_buffer
 
@@ -190,6 +241,7 @@ module Aerospike
       write_header(policy, INFO1_READ, 0, field_count, 1)
 
       write_key(key)
+      write_predexp(policy.predexp, predexp_size)
       write_operation_for_bin_name('', Aerospike::Operation::READ)
       end_cmd
     end
@@ -198,40 +250,55 @@ module Aerospike
     def set_operate(policy, key, operations)
       begin_cmd
       field_count = estimate_key_size(key, policy)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       read_attr = 0
       write_attr = 0
       read_header = false
+      record_bin_multiplicity = policy.record_bin_multiplicity == RecordBinMultiplicity::ARRAY
 
       operations.each do |operation|
         case operation.op_type
-        when Aerospike::Operation::READ
-            read_attr |= INFO1_READ
+        when Aerospike::Operation::READ, Aerospike::Operation::CDT_READ, 
+              Aerospike::Operation::HLL_READ, Aerospike::Operation::BIT_READ
+          read_attr |= INFO1_READ
 
           # Read all bins if no bin is specified.
           read_attr |= INFO1_GET_ALL unless operation.bin_name
 
         when Aerospike::Operation::READ_HEADER
-            # The server does not currently return record header data with _INFO1_NOBINDATA attribute set.
-            # The workaround is to request a non-existent bin.
-            # TODO: Fix this on server.
-            # read_attr |= _INFO1_READ | _INFO1_NOBINDATA
-            read_attr |= INFO1_READ
+          # The server does not currently return record header data with _INFO1_NOBINDATA attribute set.
+          # The workaround is to request a non-existent bin.
+          # TODO: Fix this on server.
+          # read_attr |= _INFO1_READ | _INFO1_NOBINDATA
+          read_attr |= INFO1_READ
           read_header = true
 
         else
           write_attr = INFO2_WRITE
         end
 
+        if [Aerospike::Operation::HLL_MODIFY, Aerospike::Operation::HLL_READ, 
+              Aerospike::Operation::BIT_MODIFY, Aerospike::Operation::BIT_READ].include?(operation.op_type)
+          record_bin_multiplicity = true
+        end
+
         estimate_operation_size_for_operation(operation)
       end
       size_buffer
 
+
+      write_attr |= INFO2_RESPOND_ALL_OPS if write_attr != 0 && record_bin_multiplicity
+
       if write_attr != 0
         write_header_with_policy(policy, read_attr, write_attr, field_count, operations.length)
       else
         write_header(policy, read_attr, write_attr, field_count, operations.length)
       end
       write_key(key, policy)
+      write_predexp(policy.predexp, predexp_size)
 
       operations.each do |operation|
         write_operation_for_operation(operation)
@@ -240,11 +307,16 @@ module Aerospike
       write_operation_for_bin(nil, Aerospike::Operation::READ) if read_header
 
       end_cmd
+      mark_compressed(policy)
     end
 
     def set_udf(policy, key, package_name, function_name, args)
       begin_cmd
       field_count = estimate_key_size(key, policy)
+      
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
+
       arg_bytes = args.to_bytes
 
       field_count += estimate_udf_size(package_name, function_name, arg_bytes)
@@ -252,11 +324,13 @@ module Aerospike
 
       write_header(policy, 0, INFO2_WRITE, field_count, 0)
       write_key(key, policy)
+      write_predexp(policy.predexp, predexp_size)
       write_field_string(package_name, Aerospike::FieldType::UDF_PACKAGE_NAME)
       write_field_string(function_name, Aerospike::FieldType::UDF_FUNCTION)
       write_field_bytes(arg_bytes, Aerospike::FieldType::UDF_ARGLIST)
 
       end_cmd
+      mark_compressed(policy)
     end
 
     def set_scan(policy, namespace, set_name, bin_names)
@@ -273,6 +347,14 @@ module Aerospike
         @data_offset += set_name.bytesize + FIELD_HEADER_SIZE
         field_count += 1
       end
+      
+      if policy.records_per_second > 0
+        @data_offset += 4 + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      predexp_size = estimate_predexp(policy.predexp)
+      field_count += 1 if predexp_size > 0
 
       # Estimate scan options size.
       @data_offset += 2 + FIELD_HEADER_SIZE
@@ -310,6 +392,12 @@ module Aerospike
         write_field_string(set_name, Aerospike::FieldType::TABLE)
       end
 
+      if policy.records_per_second > 0
+        write_field_int(policy.records_per_second, Aerospike::FieldType::RECORDS_PER_SECOND)
+      end
+
+      write_predexp(policy.predexp, predexp_size)
+
       write_field_header(2, Aerospike::FieldType::SCAN_OPTIONS)
 
       priority = policy.priority & 0xFF
@@ -355,6 +443,7 @@ module Aerospike
         break if @policy.timeout > 0 && Time.now > limit
 
         begin
+          @node = get_node
           @conn = @node.get_connection(@policy.timeout)
         rescue => e
           # Socket connection error has occurred. Decrease health and retry.
@@ -498,9 +587,20 @@ module Aerospike
       @data_offset += OPERATION_HEADER_SIZE
     end
 
+    def estimate_predexp(predexp)
+      if predexp && predexp.size > 0
+        @data_offset += FIELD_HEADER_SIZE
+        sz = Aerospike::PredExp.estimate_size(predexp)
+        @data_offset += sz
+        return sz
+      end
+      return 0
+    end
+
     # Generic header write.
     def write_header(policy, read_attr, write_attr, field_count, operation_count)
       read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
+      read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
 
       # Write all header data except total size which must be written last.
       @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -550,6 +650,7 @@ module Aerospike
       info_attr |= INFO3_COMMIT_MASTER if policy.commit_level == Aerospike::CommitLevel::COMMIT_MASTER
       read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
       write_attr |= INFO2_DURABLE_DELETE if policy.durable_delete
+      read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
 
       # Write all header data except total size which must be written last.
       @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -681,21 +782,34 @@ module Aerospike
       @data_offset += len
     end
 
+    def write_field_int(i, ftype)
+      @data_buffer.write_int32(i, @data_offset+FIELD_HEADER_SIZE)
+      write_field_header(4, ftype)
+      @data_offset += 4
+    end
+
     def write_field_bytes(bytes, ftype)
       @data_buffer.write_binary(bytes, @data_offset+FIELD_HEADER_SIZE)
-
       write_field_header(bytes.bytesize, ftype)
       @data_offset += bytes.bytesize
     end
 
     def write_field_header(size, ftype)
-      # Buffer.Int32ToBytes(size+1), @data_buffer, @data_offset
       @data_buffer.write_int32(size+1, @data_offset)
       @data_offset += 4
       @data_buffer.write_byte(ftype, @data_offset)
       @data_offset += 1
     end
 
+    def write_predexp(predexp, predexp_size)
+      if predexp && predexp.size > 0
+        write_field_header(predexp_size, Aerospike::FieldType::PREDEXP)
+        @data_offset = Aerospike::PredExp.write(
+          predexp, @data_buffer, @data_offset
+        )
+      end
+    end
+
     def begin_cmd
       @data_offset = MSG_TOTAL_HEADER_SIZE
     end
@@ -719,6 +833,48 @@ module Aerospike
       @data_buffer.write_int64(size, 0)
     end
 
+    def use_compression?
+      @compress
+    end
+
+    def compress_buffer
+      if @data_offset > COMPRESS_THRESHOLD
+        compressed = Zlib::deflate(@data_buffer.buf, Zlib::DEFAULT_COMPRESSION)
+
+        # write original size as header
+        proto_s = "%08d" % 0
+        proto_s[0, 8] = [@data_offset].pack('q>')
+        compressed.prepend(proto_s)
+
+        # write proto
+        proto = (compressed.size+8) | Integer(CL_MSG_VERSION << 56) | Integer(AS_MSG_TYPE << 48)
+        proto_s = "%08d" % 0
+        proto_s[0, 8] = [proto].pack('q>')
+        compressed.prepend(proto_s)
+
+        @data_buffer = Buffer.new(-1, compressed)
+        @data_offset = @data_buffer.size
+      end
+    end
+
+    # isCompressed returns the length of the compressed buffer.
+    # If the buffer is not compressed, the result will be -1
+    def compressed_size
+      # A number of these are commented out because we just don't care enough to read
+      # that section of the header. If we do care, uncomment and check!
+      proto = @data_buffer.read_int64(0)
+      size = proto & 0xFFFFFFFFFFFF
+      msg_type = (proto >> 48) & 0xFF
+
+      return nil if msg_type != AS_MSG_TYPE_COMPRESSED
+
+      size
+    end
+
+    def mark_compressed(policy)
+      @compress = policy.use_compression
+    end
+
   end # class
 
 end # module