aerospike 2.19.0 → 2.26.0

data/lib/aerospike/query/query_partition_command.rb

@@ -0,0 +1,267 @@
+# encoding: utf-8
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License") you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require "aerospike/query/stream_command"
+require "aerospike/query/recordset"
+
+module Aerospike
+  private
+
+  class QueryPartitionCommand < QueryCommand #:nodoc:
+    def initialize(node, tracker, policy, statement, recordset, node_partitions)
+      super(node, policy, statement, recordset, @node_partitions)
+      @node_partitions = node_partitions
+      @tracker = tracker
+    end
+
+    def write_buffer
+      function_arg_buffer = nil
+      field_count = 0
+      filter_size = 0
+      bin_name_size = 0
+
+      begin_cmd
+
+      if @statement.namespace
+        @data_offset += @statement.namespace.bytesize + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      if @statement.set_name
+        @data_offset += @statement.set_name.bytesize + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      # Estimate recordsPerSecond field size. This field is used in new servers and not used
+      # (but harmless to add) in old servers.
+      if @policy.records_per_second > 0
+        @data_offset += 4 + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      # Estimate socket timeout field size. This field is used in new servers and not used
+      # (but harmless to add) in old servers.
+      @data_offset += 4 + FIELD_HEADER_SIZE
+      field_count += 1
+
+      # Estimate task_id field.
+      @data_offset += 8 + FIELD_HEADER_SIZE
+      field_count += 1
+
+      filter = @statement.filters[0]
+      bin_names = @statement.bin_names
+      packed_ctx = nil
+
+      if filter
+        col_type = filter.collection_type
+
+        # Estimate INDEX_TYPE field.
+        if col_type > 0
+          @data_offset += FIELD_HEADER_SIZE + 1
+          field_count += 1
+        end
+
+        # Estimate INDEX_RANGE field.
+        @data_offset += FIELD_HEADER_SIZE
+        filter_size += 1  # num filters
+        filter_size += filter.estimate_size
+
+        @data_offset += filter_size
+        field_count += 1
+
+        packed_ctx = filter.packed_ctx
+        if packed_ctx
+          @data_offset += FIELD_HEADER_SIZE + packed_ctx.length
+          field_count += 1
+        end
+      end
+
+      @statement.set_task_id
+      predexp = @policy.predexp || @statement.predexp
+
+      if predexp
+        @data_offset += FIELD_HEADER_SIZE
+        pred_size = Aerospike::PredExp.estimate_size(predexp)
+        @data_offset += pred_size
+        field_count += 1
+      end
+
+      unless @policy.filter_exp.nil?
+        exp_size = estimate_expression_size(@policy.filter_exp)
+        field_count += 1 if exp_size > 0
+      end
+
+      # Estimate aggregation/background function size.
+      if @statement.function_name
+        @data_offset += FIELD_HEADER_SIZE + 1 # udf type
+        @data_offset += @statement.package_name.bytesize + FIELD_HEADER_SIZE
+        @data_offset += @statement.function_name.bytesize + FIELD_HEADER_SIZE
+
+        function_arg_buffer = ""
+        if @statement.function_args && @statement.function_args.length > 0
+          function_arg_buffer = Value.of(@statement.function_args).to_bytes
+        end
+        @data_offset += FIELD_HEADER_SIZE + function_arg_buffer.bytesize
+        field_count += 4
+      end
+
+      max_records = 0
+      parts_full_size = 0
+      parts_partial_digest_size = 0
+      parts_partial_bval_size = 0
+
+      unless @node_partitions.nil?
+        parts_full_size = @node_partitions.parts_full.length * 2
+        parts_partial_digest_size = @node_partitions.parts_partial.length * 20
+
+        unless filter.nil?
+          parts_partial_bval_size = @node_partitions.parts_partial.length * 8
+        end
+        max_records = @node_partitions.record_max
+      end
+
+      if parts_full_size > 0
+        @data_offset += parts_full_size + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      if parts_partial_digest_size > 0
+        @data_offset += parts_partial_digest_size + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      if parts_partial_bval_size > 0
+        @data_offset += parts_partial_bval_size + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      # Estimate max records field size. This field is used in new servers and not used
+      # (but harmless to add) in old servers.
+      if max_records > 0
+        @data_offset += 8 + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      operation_count = 0
+      unless bin_names.empty?
+        # Estimate size for selected bin names (query bin names already handled for old servers).
+        bin_names.each do |bin_name|
+          estimate_operation_size_for_bin_name(bin_name)
+        end
+        operation_count = bin_names.length
+      end
+
+      projected_offset = @data_offset
+
+      size_buffer
+
+      read_attr = INFO1_READ
+      read_attr |= INFO1_NOBINDATA if !@policy.include_bin_data
+      read_attr |= INFO1_SHORT_QUERY if @policy.short_query
+
+      infoAttr = INFO3_PARTITION_DONE
+
+      write_header(@policy, read_attr, 0, field_count, operation_count)
+
+      write_field_string(@statement.namespace, FieldType::NAMESPACE) if @statement.namespace
+      write_field_string(@statement.set_name, FieldType::TABLE) if @statement.set_name
+
+      # Write records per second.
+      write_field_int(@policy.records_per_second, FieldType::RECORDS_PER_SECOND) if @policy.records_per_second > 0
+
+      write_filter_exp(@policy.filter_exp, exp_size)
+
+      # Write socket idle timeout.
+      write_field_int(@policy.socket_timeout, FieldType::SOCKET_TIMEOUT)
+
+      # Write task_id field
+      write_field_int64(@statement.task_id, FieldType::TRAN_ID)
+
+      unless predexp.nil?
+        write_field_header(pred_size, Aerospike::FieldType::PREDEXP)
+        @data_offset = Aerospike::PredExp.write(
+          predexp, @data_buffer, @data_offset
+        )
+      end
+
+      if filter
+        type = filter.collection_type
+
+        if type > 0
+          write_field_header(1, FieldType::INDEX_TYPE)
+          @data_offset += @data_buffer.write_byte(type, @data_offset)
+        end
+
+        write_field_header(filter_size, FieldType::INDEX_RANGE)
+        @data_offset += @data_buffer.write_byte(1, @data_offset)
+        @data_offset = filter.write(@data_buffer, @data_offset)
+
+        if packed_ctx
+          write_field_header(packed_ctx.length, FieldType::INDEX_CONTEXT)
+          @data_offset += @data_buffer.write_binary(packed_ctx, @data_offset)
+        end
+      end
+
+      if @statement.function_name
+        write_field_header(1, FieldType::UDF_OP)
+        @data_offset += @data_buffer.write_byte(1, @data_offset)
+        write_field_string(@statement.package_name, FieldType::UDF_PACKAGE_NAME)
+        write_field_string(@statement.function_name, FieldType::UDF_FUNCTION)
+        write_field_string(function_arg_buffer, FieldType::UDF_ARGLIST)
+      end
+
+      if parts_full_size > 0
+        write_field_header(parts_full_size, FieldType::PID_ARRAY)
+        @node_partitions.parts_full.each do |part|
+          @data_offset += @data_buffer.write_uint16_little_endian(part.id, @data_offset)
+        end
+      end
+
+      if parts_partial_digest_size > 0
+        write_field_header(parts_partial_digest_size, FieldType::DIGEST_ARRAY)
+        @node_partitions.parts_partial.each do |part|
+          @data_offset += @data_buffer.write_binary(part.digest, @data_offset)
+        end
+      end
+
+      if parts_partial_bval_size > 0
+        write_field_header(parts_partial_bval_size, FieldType::BVAL_ARRAY)
+        @node_partitions.parts_partial.each do |part|
+          @data_offset += @data_buffer.write_uint64_little_endian(part.bval, @data_offset)
+        end
+      end
+
+      if max_records > 0
+        write_field(max_records, FieldType::MAX_RECORDS)
+      end
+
+      unless bin_names.empty?
+        bin_names.each do |bin_name|
+          write_operation_for_bin_name(bin_name, Operation::READ)
+        end
+      end
+
+      end_cmd
+
+      nil
+    end
+
+    def should_retry(e)
+      # !! converts nil to false
+      !!@tracker&.should_retry(@node_partitions, e)
+    end
+  end # class
+end # module

data/lib/aerospike/query/recordset.rb

@@ -22,7 +22,6 @@ module Aerospike
   # so the production and the consumptoin are decoupled
   # there can be an unlimited count of producer threads and consumer threads
   class Recordset
-
     attr_reader :records
 
     def initialize(queue_size = 5000, thread_count = 1, type)
@@ -66,18 +65,21 @@ module Aerospike
 
     # this is called by working threads to signal their job is finished
     # it decreases the count of active threads and puts an EOF on queue when all threads are finished
-    def thread_finished
+    # e is an exception that has happened in the exceutor, and outside of the threads themselves
+    def thread_finished(expn = nil)
       @active_threads.update do |v|
         v -= 1
         @records.enq(nil) if v == 0
         v
       end
+
+      raise expn unless expn.nil?
     end
 
     # this is called by a thread who faced an exception to singnal to terminate the whole operation
     # it also may be called by the user to terminate the command in the middle of fetching records from server nodes
     # it clears the queue so that if any threads are waiting for the queue get unblocked and find out about the cancellation
-    def cancel(expn=nil)
+    def cancel(expn = nil)
       set_exception(expn)
       @cancelled.set(true)
       @records.clear
@@ -104,18 +106,16 @@ module Aerospike
       @filters.nil? || @filters.empty?
     end
 
-  private
+    private
 
-    def set_exception(expn=nil)
+    def set_exception(expn = nil)
       expn ||= (@type == :scan ? SCAN_TERMINATED_EXCEPTION : QUERY_TERMINATED_EXCEPTION)
       @thread_exception.set(expn)
     end
-
   end
 
   private
 
-    SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
-    QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
-
+  SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
+  QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
 end

data/lib/aerospike/query/scan_command.rb

@@ -23,7 +23,7 @@ module Aerospike
 
   class ScanCommand < StreamCommand #:nodoc:
 
-    def initialize(node, policy, namespace, set_name, bin_names, recordset)
+    def initialize(node, policy, namespace, set_name, bin_names, recordset, node_partitions)
       super(node)
 
       @policy = policy
@@ -31,10 +31,11 @@ module Aerospike
       @set_name = set_name
       @bin_names = bin_names
       @recordset = recordset
+      @node_partitions = node_partitions
     end
 
     def write_buffer
-      set_scan(@policy, @namespace, @set_name, @bin_names)
+      set_scan(@policy, @namespace, @set_name, @bin_names, @node_partitions)
     end
 
   end # class

data/lib/aerospike/query/scan_executor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module Aerospike
+  class ScanExecutor # :nodoc:
+    def self.scan_partitions(policy, cluster, tracker, namespace, set_name, recordset, bin_names = nil)
+      interval = policy.sleep_between_retries
+
+      should_retry = false
+
+      loop do
+        # reset last_expn
+        @last_expn = nil
+
+        list = tracker.assign_partitions_to_nodes(cluster, namespace)
+
+        if policy.concurrent_nodes
+          threads = []
+          # Use a thread per node
+          list.each do |node_partition|
+            threads << Thread.new do
+              Thread.current.abort_on_exception = true
+              command = ScanPartitionCommand.new(policy, tracker, node_partition, namespace, set_name, bin_names, recordset)
+              begin
+                command.execute
+              rescue => e
+                @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
+                should_retry ||= command.should_retry(e)
+                Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
+              end
+            end
+          end
+          threads.each(&:join)
+        else
+          # Use a single thread for all nodes for all node
+          list.each do |node_partition|
+            command = ScanPartitionCommand.new(policy, tracker, node_partition, namespace, set_name, bin_names, recordset)
+            begin
+              command.execute
+            rescue => e
+              @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
+              should_retry ||= command.should_retry(e)
+              Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
+            end
+          end
+        end
+
+        if tracker.complete?(@cluster, policy) || !should_retry
+          recordset.thread_finished(@last_expn)
+          return
+        end
+        sleep(interval) if policy.sleep_between_retries > 0
+        statement.reset_task_id
+      end
+    end
+  end
+end

data/lib/aerospike/query/scan_partition_command.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'aerospike/query/stream_command'
+require 'aerospike/query/recordset'
+
+module Aerospike
+
+  private
+
+  class ScanPartitionCommand < StreamCommand #:nodoc:
+
+    def initialize(policy, tracker, node_partitions, namespace, set_name, bin_names, recordset)
+      super(node_partitions.node)
+
+      @policy = policy
+      @namespace = namespace
+      @set_name = set_name
+      @bin_names = bin_names
+      @recordset = recordset
+      @node_partitions = node_partitions
+      @tracker = tracker
+    end
+
+    def write_buffer
+      set_scan(@policy, @namespace, @set_name, @bin_names, @node_partitions)
+    end
+
+    def should_retry(e)
+      # !! converts nil to false
+      !!@tracker&.should_retry(@node_partitions, e)
+    end
+
+  end # class
+
+end # module

data/lib/aerospike/query/statement.rb

@@ -49,7 +49,7 @@ module Aerospike
       # This method is redundant because PredExp can now be set in the base Policy for
       # any transaction (including queries).
       #
-      # NOTE : Policy.predexp takes precedence to this value. This value will be 
+      # NOTE : Policy.predexp takes precedence to this value. This value will be
       # deprecated in the future.
       @predexp = nil
 
@@ -81,6 +81,13 @@ module Aerospike
     end
   end
 
+  def reset_task_id
+    @task_id = rand(RAND_MAX)
+    while @task_id == 0
+      @task_id = rand(RAND_MAX)
+    end
+  end
+
   private
 
   RAND_MAX = 2**63

data/lib/aerospike/query/stream_command.rb

@@ -39,6 +39,8 @@ module Aerospike
         case result_code
         when Aerospike::ResultCode::OK
           # noop
+        when Aerospike::ResultCode::PARTITION_UNAVAILABLE
+          # noop
         when Aerospike::ResultCode::KEY_NOT_FOUND_ERROR
           # consume the rest of the input buffer from the socket
           read_bytes(receive_size - @data_offset) if @data_offset < receive_size
@@ -58,6 +60,18 @@ module Aerospike
         op_count = @data_buffer.read_int16(20)
         key = parse_key(field_count)
 
+        # If cmd is the end marker of the response, do not proceed further
+        if (info3 & INFO3_PARTITION_DONE) != 0
+          # When an error code is received, mark partition as unavailable
+          # for the current round. Unavailable partitions will be retried
+          # in the next round. Generation is overloaded as partitionId.
+          if result_code != 0
+            @tracker&.partition_unavailable(@node_partitions, generation)
+          end
+
+          next
+        end
+
         if result_code == 0
           if @recordset.active?
             @recordset.records.enq(parse_record(key, op_count, generation, expiration))
@@ -65,6 +79,9 @@ module Aerospike
             expn = @recordset.is_scan? ? SCAN_TERMINATED_EXCEPTION : QUERY_TERMINATED_EXCEPTION
             raise expn
           end
+
+          # UDF results do not return a key
+          @tracker&.set_last(@node_partitions, key, key.bval) if key
         end
       end # while
 

data/lib/aerospike/result_code.rb

@@ -20,6 +20,42 @@ module Aerospike
 
     attr_reader :code
 
+    # One or more keys failed in a batch.
+    BATCH_FAILED = -20
+
+    # No response was received from the server.
+    NO_RESPONSE = -19
+
+    # A network error. Checked the wrapped error for detail.
+    NETWORK_ERROR = -18
+
+    # A common, none-aerospike error. Checked the wrapped error for detail.
+    COMMON_ERROR = -17
+
+    # Max retries limit reached.
+    MAX_RETRIES_EXCEEDED = -16
+
+    # Max errors limit reached.
+    MAX_ERROR_RATE = -15
+
+    # Requested Rack for node/namespace was not defined in the cluster.
+    RACK_NOT_DEFINED = -13
+
+    # Cluster has an invalid partition map, usually due to bad configuration.
+    INVALID_CLUSTER_PARTITION_MAP = -12
+
+    # Server is not accepting requests.
+    SERVER_NOT_AVAILABLE = -11
+
+    # Cluster Name does not match the ClientPolicy.ClusterName value.
+    CLUSTER_NAME_MISMATCH_ERROR = -10
+
+    # Recordset has already been closed or cancelled
+    RECORDSET_CLOSED = -9
+
+    # There were no connections available to the node in the pool, and the pool was limited
+    NO_AVAILABLE_CONNECTIONS_TO_NODE = -8
+
     # Value type not supported by Aerospike server
     TYPE_NOT_SUPPORTED = -7
 
@@ -76,8 +112,8 @@ module Aerospike
     # XDS product is not available.
     NO_XDS = 10
 
-    # Server is not accepting requests.
-    SERVER_NOT_AVAILABLE = 11
+    # Partition is unavailable.
+    PARTITION_UNAVAILABLE = 11
 
     # Operation is not supported with configured bin type (single-bin or
     # multi-bin).
@@ -182,7 +218,7 @@ module Aerospike
     # Privilege is invalid.
     INVALID_PRIVILEGE = 72
 
-    # Specified IP whitelist is invalid.
+    # Specified IP allowlist is invalid.
     INVALID_WHITELIST = 73
 
     # User must be authentication before performing database operations.
@@ -191,7 +227,7 @@ module Aerospike
     # User does not posses the required role to perform the database operation.
     ROLE_VIOLATION = 81
 
-    # Client IP address is not on the IP whitelist.
+    # Client IP address is not on the IP allowlist.
     NOT_WHITELISTED = 82
 
     # LDAP feature not enabled on server.
@@ -265,6 +301,45 @@ module Aerospike
 
     def self.message(code)
       case code
+      when BATCH_FAILED
+          "one or more keys failed in a batch"
+
+      when NO_RESPONSE
+          "no response was received from the server"
+
+      when NETWORK_ERROR
+          "network error. Checked the wrapped error for detail"
+
+      when COMMON_ERROR
+          "common, none-aerospike error. Checked the wrapped error for detail"
+
+      when MAX_RETRIES_EXCEEDED
+          "Max retries exceeded"
+
+      when MAX_ERROR_RATE
+          "Max errors limit reached for node"
+
+      when RACK_NOT_DEFINED
+          "Requested Rack for node/namespace was not defined in the cluster."
+
+      when INVALID_CLUSTER_PARTITION_MAP
+          "Cluster has an invalid partition map, usually due to bad configuration."
+
+      when SERVER_NOT_AVAILABLE
+          "Server is not accepting requests."
+
+      when CLUSTER_NAME_MISMATCH_ERROR
+          "Cluster Name does not match the ClientPolicy.ClusterName value"
+
+      when RECORDSET_CLOSED
+          "Recordset has already been closed or cancelled."
+
+      when NO_AVAILABLE_CONNECTIONS_TO_NODE
+          "No available connections to the node. Connection Pool was empty, and limited to certain number of connections."
+
+      when TYPE_NOT_SUPPORTED
+          "Type cannot be converted to Value Type."
+
       when COMMAND_REJECTED
         "Command rejected"
 
@@ -316,8 +391,8 @@ module Aerospike
       when NO_XDS
         "XDS not available"
 
-      when SERVER_NOT_AVAILABLE
-        "Server not available"
+      when PARTITION_UNAVAILABLE
+          "Partition not available"
 
       when BIN_TYPE_ERROR
         "Bin type error"
@@ -422,7 +497,7 @@ module Aerospike
         "Invalid privilege"
 
       when INVALID_WHITELIST
-        "Specified IP whitelist is invalid"
+        "Specified IP allowlist is invalid"
 
       when NOT_AUTHENTICATED
         "Not authenticated"
@@ -431,7 +506,7 @@ module Aerospike
         "Role violation"
 
       when NOT_WHITELISTED
-        "Client IP address is not on the IP whitelist"
+        "Client IP address is not on the IP allowlist"
 
       when LDAP_NOT_ENABLED
         "LDAP feature not enabled on server"