aerospike 2.12.0 → 2.17.0

data/lib/aerospike/cluster/rack_parser.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'base64'
+
+module Aerospike
+
+  class RackParser #:nodoc:
+
+    attr_accessor :rebalance_generation, :racks
+
+    REBALANCE_GENERATION = "rebalance-generation"
+    RACK_IDS = "rack-ids"
+
+    def initialize(node, conn)
+      @node = node
+      @conn = conn
+      @racks = nil
+    end
+
+    def update_racks
+      # Use low-level info methods and parse byte array directly for maximum performance.
+      # Receive format: rack-ids\t
+      #                 <ns1>:<rack-id>...;
+      #                 <ns2>:<rack-id>...; ...
+      info_map = Info.request(@conn, REBALANCE_GENERATION, RACK_IDS)
+
+      @rebalance_generation = info_map[REBALANCE_GENERATION].to_i
+
+      info = info_map[RACK_IDS]
+      if !info || info.length == 0
+        raise Aerospike::Exceptions::Connection.new("#{RACK_IDS} response for node #{@node.name} is empty")
+      end
+
+      @buffer = info
+      @length = info.length
+      @offset = 0
+
+      while @offset < @length && @buffer[@offset] != '\n'
+        namespace = parse_name
+        rack_id = parse_rack_id
+
+        @racks = {} if !@racks
+        @racks[namespace] = rack_id
+      end
+
+      @racks
+    end
+
+    private
+
+    def parse_name
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ':'
+        @offset+=1
+      end
+
+      # Parse namespace.
+      namespace = @buffer[beginning...@offset].strip
+
+      if namespace.length <= 0 || namespace.length >= 32
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid rack namespace #{namespace}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      namespace
+    end
+
+    def parse_rack_id
+      beginning = @offset
+      while @offset < @length
+        break if @buffer[@offset] == ';'
+        @offset+=1
+      end
+
+      # Parse rack_id
+      rack_id = @buffer[beginning...@offset].strip.to_i
+
+      if rack_id < 0
+        response = get_truncated_response
+        raise Aerospike::Exceptions::Parse.new(
+          "Invalid rack_id #{rack_id}. Response=#{response}"
+        )
+      end
+
+      @offset+=1
+      rack_id
+    end
+
+    def get_truncated_response
+      max = @length
+      @length = max if @length > 200
+      @buffer[0...max]
+    end
+
+
+  end # class
+
+end # module

data/lib/aerospike/command/admin_command.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/batch_direct_command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -75,6 +75,7 @@ module Aerospike
       end
 
       end_cmd
+      mark_compressed(@policy)
     end
 
     # Parse all results in the batch.  Add records to shared list.

data/lib/aerospike/command/batch_direct_exists_command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/batch_direct_node.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -24,8 +24,8 @@ module Aerospike
     attr_accessor :node
     attr_accessor :batch_namespaces
 
-    def self.generate_list(cluster, keys)
-      keys.group_by { |key| cluster.get_node_for_key(key) }
+    def self.generate_list(cluster, replica_policy, keys)
+      keys.group_by { |key| cluster.get_node_for_key(replica_policy, key) }
         .map { |node, keys_for_node| BatchDirectNode.new(node, keys_for_node) }
     end
 

data/lib/aerospike/command/batch_index_command.rb

@@ -98,6 +98,7 @@ module Aerospike
         end
       end
       end_cmd
+      mark_compressed(@policy)
     end
 
     # Parse all results in the batch.  Add records to shared list.

data/lib/aerospike/command/batch_index_node.rb

@@ -22,9 +22,9 @@ module Aerospike
     attr_accessor :node
     attr_accessor :keys_by_idx
 
-    def self.generate_list(cluster, keys)
+    def self.generate_list(cluster, replica_policy, keys)
       keys.each_with_index
-        .group_by { |key, _| cluster.get_node_for_key(key) }
+        .group_by { |key, _| cluster.get_node_for_key(replica_policy, key) }
         .map { |node, keys_with_idx| BatchIndexNode.new(node, keys_with_idx) }
     end
 

data/lib/aerospike/command/batch_item.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/command/command.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -16,6 +16,7 @@
 # the License.
 
 require 'time'
+require 'zlib'
 
 require 'msgpack'
 require 'aerospike/result_code'
@@ -42,6 +43,9 @@ module Aerospike
   # Involve all replicas in read operation.
   INFO1_CONSISTENCY_ALL = Integer(1 << 6)
 
+  # Tell server to compress it's response.
+  INFO1_COMPRESS_RESPONSE = (1 << 7)
+
   # Create or update record
   INFO2_WRITE = Integer(1 << 0)
   # Fling a record into the belly of Moloch.
@@ -55,6 +59,9 @@ module Aerospike
   # Create only. Fail if record already exists.
   INFO2_CREATE_ONLY = Integer(1 << 5)
 
+  # Return a result for every operation.
+  INFO2_RESPOND_ALL_OPS = Integer(1 << 7)
+
   # This is the last of a multi-part message.
   INFO3_LAST = Integer(1 << 0)
   # Commit to master only before declaring success.
@@ -72,17 +79,24 @@ module Aerospike
   OPERATION_HEADER_SIZE      = 8
   MSG_REMAINING_HEADER_SIZE  = 22
   DIGEST_SIZE                = 20
+  COMPRESS_THRESHOLD         = 128
   CL_MSG_VERSION             = 2
   AS_MSG_TYPE                = 3
+  AS_MSG_TYPE_COMPRESSED     = 4
 
   class Command #:nodoc:
 
-    def initialize(node)
+    def initialize(node=nil)
       @data_offset = 0
       @data_buffer = nil
 
       @node = node
 
+      @compress = false
+
+      # will add before use
+      @sequence = Atomic.new(-1)
+
       self
     end
 
@@ -115,6 +129,7 @@ module Aerospike
       end
 
       end_cmd
+      mark_compressed(policy)
     end
 
     # Writes the command for delete operations
@@ -242,10 +257,12 @@ module Aerospike
       read_attr = 0
       write_attr = 0
       read_header = false
+      record_bin_multiplicity = policy.record_bin_multiplicity == RecordBinMultiplicity::ARRAY
 
       operations.each do |operation|
         case operation.op_type
-        when Aerospike::Operation::READ
+        when Aerospike::Operation::READ, Aerospike::Operation::CDT_READ, 
+              Aerospike::Operation::HLL_READ, Aerospike::Operation::BIT_READ
           read_attr |= INFO1_READ
 
           # Read all bins if no bin is specified.
@@ -259,17 +276,22 @@ module Aerospike
           read_attr |= INFO1_READ
           read_header = true
 
-        when Aerospike::Operation::CDT_READ
-          read_attr |= INFO1_READ
-
         else
           write_attr = INFO2_WRITE
         end
 
+        if [Aerospike::Operation::HLL_MODIFY, Aerospike::Operation::HLL_READ, 
+              Aerospike::Operation::BIT_MODIFY, Aerospike::Operation::BIT_READ].include?(operation.op_type)
+          record_bin_multiplicity = true
+        end
+
         estimate_operation_size_for_operation(operation)
       end
       size_buffer
 
+
+      write_attr |= INFO2_RESPOND_ALL_OPS if write_attr != 0 && record_bin_multiplicity
+
       if write_attr != 0
         write_header_with_policy(policy, read_attr, write_attr, field_count, operations.length)
       else
@@ -285,6 +307,7 @@ module Aerospike
       write_operation_for_bin(nil, Aerospike::Operation::READ) if read_header
 
       end_cmd
+      mark_compressed(policy)
     end
 
     def set_udf(policy, key, package_name, function_name, args)
@@ -307,6 +330,7 @@ module Aerospike
       write_field_bytes(arg_bytes, Aerospike::FieldType::UDF_ARGLIST)
 
       end_cmd
+      mark_compressed(policy)
     end
 
     def set_scan(policy, namespace, set_name, bin_names)
@@ -324,6 +348,11 @@ module Aerospike
         field_count += 1
       end
       
+      if policy.records_per_second > 0
+        @data_offset += 4 + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
       predexp_size = estimate_predexp(policy.predexp)
       field_count += 1 if predexp_size > 0
 
@@ -363,6 +392,10 @@ module Aerospike
         write_field_string(set_name, Aerospike::FieldType::TABLE)
       end
 
+      if policy.records_per_second > 0
+        write_field_int(policy.records_per_second, Aerospike::FieldType::RECORDS_PER_SECOND)
+      end
+
       write_predexp(policy.predexp, predexp_size)
 
       write_field_header(2, Aerospike::FieldType::SCAN_OPTIONS)
@@ -410,6 +443,7 @@ module Aerospike
         break if @policy.timeout > 0 && Time.now > limit
 
         begin
+          @node = get_node
           @conn = @node.get_connection(@policy.timeout)
         rescue => e
           # Socket connection error has occurred. Decrease health and retry.
@@ -566,6 +600,7 @@ module Aerospike
     # Generic header write.
     def write_header(policy, read_attr, write_attr, field_count, operation_count)
       read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
+      read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
 
       # Write all header data except total size which must be written last.
       @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -615,6 +650,7 @@ module Aerospike
       info_attr |= INFO3_COMMIT_MASTER if policy.commit_level == Aerospike::CommitLevel::COMMIT_MASTER
       read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
       write_attr |= INFO2_DURABLE_DELETE if policy.durable_delete
+      read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
 
       # Write all header data except total size which must be written last.
       @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -746,15 +782,19 @@ module Aerospike
       @data_offset += len
     end
 
+    def write_field_int(i, ftype)
+      @data_buffer.write_int32(i, @data_offset+FIELD_HEADER_SIZE)
+      write_field_header(4, ftype)
+      @data_offset += 4
+    end
+
     def write_field_bytes(bytes, ftype)
       @data_buffer.write_binary(bytes, @data_offset+FIELD_HEADER_SIZE)
-
       write_field_header(bytes.bytesize, ftype)
       @data_offset += bytes.bytesize
     end
 
     def write_field_header(size, ftype)
-      # Buffer.Int32ToBytes(size+1), @data_buffer, @data_offset
       @data_buffer.write_int32(size+1, @data_offset)
       @data_offset += 4
       @data_buffer.write_byte(ftype, @data_offset)
@@ -793,6 +833,48 @@ module Aerospike
       @data_buffer.write_int64(size, 0)
     end
 
+    def use_compression?
+      @compress
+    end
+
+    def compress_buffer
+      if @data_offset > COMPRESS_THRESHOLD
+        compressed = Zlib::deflate(@data_buffer.buf, Zlib::DEFAULT_COMPRESSION)
+
+        # write original size as header
+        proto_s = "%08d" % 0
+        proto_s[0, 8] = [@data_offset].pack('q>')
+        compressed.prepend(proto_s)
+
+        # write proto
+        proto = (compressed.size+8) | Integer(CL_MSG_VERSION << 56) | Integer(AS_MSG_TYPE << 48)
+        proto_s = "%08d" % 0
+        proto_s[0, 8] = [proto].pack('q>')
+        compressed.prepend(proto_s)
+
+        @data_buffer = Buffer.new(-1, compressed)
+        @data_offset = @data_buffer.size
+      end
+    end
+
+    # isCompressed returns the length of the compressed buffer.
+    # If the buffer is not compressed, the result will be -1
+    def compressed_size
+      # A number of these are commented out because we just don't care enough to read
+      # that section of the header. If we do care, uncomment and check!
+      proto = @data_buffer.read_int64(0)
+      size = proto & 0xFFFFFFFFFFFF
+      msg_type = (proto >> 48) & 0xFF
+
+      return nil if msg_type != AS_MSG_TYPE_COMPRESSED
+
+      size
+    end
+
+    def mark_compressed(policy)
+      @compress = policy.use_compression
+    end
+
   end # class
 
 end # module

data/lib/aerospike/command/delete_command.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -33,6 +33,10 @@ module Aerospike
       self
     end
 
+    def get_node
+      @cluster.master_node(@partition)
+    end
+
     def write_buffer
       set_delete(@policy, @key)
     end

data/lib/aerospike/command/execute_command.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.