aerospike 2.23.0 → 2.25.0

data/lib/aerospike/policy/policy.rb

@@ -13,26 +13,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'aerospike/policy/priority'
-require 'aerospike/policy/consistency_level'
-require 'aerospike/policy/replica'
-
+require "aerospike/policy/priority"
+require "aerospike/policy/consistency_level"
+require "aerospike/policy/replica"
 
 module Aerospike
 
   # Container object for client policy command.
   class Policy
-
-    attr_accessor :priority, :timeout, :max_retries, :sleep_between_retries, :consistency_level,
+    attr_accessor :filter_exp, :priority, :timeout, :max_retries, :sleep_between_retries, :consistency_level,
                   :predexp, :fail_on_filtered_out, :replica, :use_compression
 
     alias total_timeout timeout
     alias total_timeout= timeout=
 
-    def initialize(opt={})
+    def initialize(opt = {})
       # Container object for transaction policy attributes used in all database
       # operation calls.
 
+      # Optional expression filter. If filterExp exists and evaluates to false, the
+      # transaction is ignored.
+      #
+      # Default: nil
+      #
+      # ==== Examples:
+      #
+      # p = Policy.new
+      # p.filter_exp = Exp.build(Exp.eq(Exp.int_bin("a"), Exp.int_val(11)));
+      @filter_exp = opt[:filter_exp]
+
+      #  Throw exception if {#filter_exp} is defined and that filter evaluates
+      #  to false (transaction ignored).  The {AerospikeException}
+      #  will contain result code {ResultCode::FILTERED_OUT}.
+      #
+      #  This field is not applicable to batch, scan or query commands.
+      #
+      #  Default: false
+      @fail_on_filtered_out = opt[:fail_on_filtered_out] || false
+
       # Priority of request relative to other transactions.
       # Currently, only used for scans.
       @priority = opt[:priority] || Priority::DEFAULT
@@ -74,7 +92,6 @@ module Aerospike
       # ]
       @predexp = opt[:predexp] || nil
 
-
       # Throw exception if @predexp is defined and that filter evaluates
       # to false (transaction ignored). The Aerospike::Exceptions::Aerospike
       # will contain result code Aerospike::ResultCode::FILTERED_OUT.
@@ -86,7 +103,6 @@ module Aerospike
       # read operation.
       @consistency_level = opt[:consistency_level] || Aerospike::ConsistencyLevel::CONSISTENCY_ONE
 
-
       # Send read commands to the node containing the key's partition replica type.
       # Write commands are not affected by this setting, because all writes are directed
       # to the node containing the key's master partition.
@@ -118,8 +134,5 @@ module Aerospike
       # timeout was not exceeded. Enter zero to skip sleep.
       @sleep_between_retries = opt[:sleep_between_retries] || 0.5
     end
-
-
   end # class
-
 end # module

data/lib/aerospike/policy/query_policy.rb

@@ -22,20 +22,45 @@ module Aerospike
   # Container object for query policy command.
   class QueryPolicy < Policy
 
+    attr_accessor :concurrent_nodes
+    attr_accessor :max_records
     attr_accessor :include_bin_data
     attr_accessor :record_queue_size
     attr_accessor :records_per_second
+    attr_accessor :socket_timeout
+    attr_accessor :short_query
 
     def initialize(opt={})
       super(opt)
 
-      @max_retries = 0
-
       # Indicates if bin data is retrieved. If false, only record digests (and
       # user keys if stored on the server) are retrieved.
       # Default is true.
       @include_bin_data = opt.fetch(:include_bin_data, true)
 
+      # Approximates the number of records to return to the client. This number is divided by the
+      # number of nodes involved in the query. The actual number of records returned
+      # may be less than MaxRecords if node record counts are small and unbalanced across
+      # nodes.
+      #
+      # This field is supported on server versions >= 4.9.
+      #
+      # Default: 0 (do not limit record count)
+      @max_records = opt.fetch(:max_records) { 0 }
+
+      # Issue scan requests in parallel or serially.
+      @concurrent_nodes = opt.fetch(:concurrent_nodes) { true }
+
+      # Determines network timeout for each attempt.
+      #
+      # If socket_timeout is not zero and socket_timeout is reached before an attempt completes,
+      # the Timeout above is checked. If Timeout is not exceeded, the transaction
+      # is retried. If both socket_timeout and Timeout are non-zero, socket_timeout must be less
+      # than or equal to Timeout, otherwise Timeout will also be used for socket_timeout.
+      #
+      # Default: 30s
+      @socket_timeout = opt[:socket_timeout] || 30000
+
       # Number of records to place in queue before blocking. Records received
       # from multiple server nodes will be placed in a queue. A separate thread
       # consumes these records in parallel. If the queue is full, the producer
@@ -49,6 +74,14 @@ module Aerospike
       # Default is 0
       @records_per_second = opt[:records_per_second] || 0
 
+      # Detemine wether query expected to return less than 100 records.
+      # If true, the server will optimize the query for a small record set.
+      # This field is ignored for aggregation queries, background queries
+      # and server versions 6.0+.
+      #
+      # Default: false
+      @short_query = opt[:short_query] ||false
+
       self
     end
 

data/lib/aerospike/policy/scan_policy.rb

@@ -34,8 +34,6 @@ module Aerospike
     def initialize(opt={})
       super(opt)
 
-      @max_retries = 0
-
       # Approximates the number of records to return to the client. This number is divided by the
       # number of nodes involved in the query. The actual number of records returned
       # may be less than MaxRecords if node record counts are small and unbalanced across

data/lib/aerospike/query/query_command.rb

@@ -82,7 +82,7 @@ module Aerospike
           @data_offset += binNameSize
           fieldCount+=1
         end
-      else    
+      else
         @data_offset += @partitions.length * 2 + FIELD_HEADER_SIZE
         fieldCount += 1
 

data/lib/aerospike/query/query_executor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module Aerospike
+  class QueryExecutor # :nodoc:
+    def self.query_partitions(cluster, policy, tracker, statement, recordset)
+      interval = policy.sleep_between_retries
+
+      should_retry = false
+
+      loop do
+        # reset last_expn
+        @last_expn = nil
+
+        list = tracker.assign_partitions_to_nodes(cluster, statement.namespace)
+
+        if policy.concurrent_nodes
+          threads = []
+          # Use a thread per node
+          list.each do |node_partition|
+            threads << Thread.new do
+              Thread.current.abort_on_exception = true
+              command = QueryPartitionCommand.new(node_partition.node, tracker, policy, statement, recordset, node_partition)
+              begin
+                command.execute
+              rescue => e
+                @last_expn = e unless e == QUERY_TERMINATED_EXCEPTION
+                should_retry ||= command.should_retry(e)
+                Aerospike.logger.error(e.backtrace.join("\n")) unless e == QUERY_TERMINATED_EXCEPTION
+              end
+            end
+          end
+          threads.each(&:join)
+        else
+          # Use a single thread for all nodes for all node
+          list.each do |node_partition|
+            command = QueryPartitionCommand.new(node_partition.node, tracker, policy, statement, recordset, node_partition)
+            begin
+              command.execute
+            rescue => e
+              @last_expn = e unless e == QUERY_TERMINATED_EXCEPTION
+              should_retry ||= command.should_retry(e)
+              Aerospike.logger.error(e.backtrace.join("\n")) unless e == QUERY_TERMINATED_EXCEPTION
+            end
+          end
+        end
+
+        if tracker.complete?(@cluster, policy) || !should_retry
+          recordset.thread_finished(@last_expn)
+          return
+        end
+        sleep(interval) if policy.sleep_between_retries > 0
+        statement.reset_task_id
+      end
+    end
+  end
+end

data/lib/aerospike/query/query_partition_command.rb

@@ -0,0 +1,269 @@
+# encoding: utf-8
+# Copyright 2014-2020 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License") you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require "aerospike/query/stream_command"
+require "aerospike/query/recordset"
+
+module Aerospike
+  private
+
+  class QueryPartitionCommand < QueryCommand #:nodoc:
+    def initialize(node, tracker, policy, statement, recordset, node_partitions)
+      super(node, policy, statement, recordset, @node_partitions)
+      @node_partitions = node_partitions
+      @tracker = tracker
+    end
+
+    def write_buffer
+      function_arg_buffer = nil
+      field_count = 0
+      filter_size = 0
+      bin_name_size = 0
+
+      begin_cmd
+
+      if @statement.namespace
+        @data_offset += @statement.namespace.bytesize + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      if @statement.set_name
+        @data_offset += @statement.set_name.bytesize + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      # Estimate recordsPerSecond field size. This field is used in new servers and not used
+      # (but harmless to add) in old servers.
+      if @policy.records_per_second > 0
+        @data_offset += 4 + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      # Estimate socket timeout field size. This field is used in new servers and not used
+      # (but harmless to add) in old servers.
+      @data_offset += 4 + FIELD_HEADER_SIZE
+      field_count += 1
+
+      # Estimate task_id field.
+      @data_offset += 8 + FIELD_HEADER_SIZE
+      field_count += 1
+
+      filter = @statement.filters[0]
+      bin_names = @statement.bin_names
+      packed_ctx = nil
+
+      if filter
+        col_type = filter.collection_type
+
+        # Estimate INDEX_TYPE field.
+        if col_type > 0
+          @data_offset += FIELD_HEADER_SIZE + 1
+          field_count += 1
+        end
+
+        # Estimate INDEX_RANGE field.
+        @data_offset += FIELD_HEADER_SIZE
+        filter_size += 1  # num filters
+        filter_size += filter.estimate_size
+
+        @data_offset += filter_size
+        field_count += 1
+
+        # TODO: Implement
+        # packed_ctx = filter.packed_ctx
+        # if packed_ctx
+        #   @data_offset += FIELD_HEADER_SIZE + packed_ctx.length
+        #   field_count+=1
+        # end
+      end
+
+      @statement.set_task_id
+      predexp = @policy.predexp || @statement.predexp
+
+      if predexp
+        @data_offset += FIELD_HEADER_SIZE
+        pred_size = Aerospike::PredExp.estimate_size(predexp)
+        @data_offset += pred_size
+        field_count += 1
+      end
+
+      unless @policy.filter_exp.nil?
+        exp_size = estimate_expression_size(@policy.filter_exp)
+        field_count += 1 if exp_size > 0
+      end
+
+      # Estimate aggregation/background function size.
+      if @statement.function_name
+        @data_offset += FIELD_HEADER_SIZE + 1 # udf type
+        @data_offset += @statement.package_name.bytesize + FIELD_HEADER_SIZE
+        @data_offset += @statement.function_name.bytesize + FIELD_HEADER_SIZE
+
+        function_arg_buffer = ""
+        if @statement.function_args && @statement.function_args.length > 0
+          function_arg_buffer = Value.of(@statement.function_args).to_bytes
+        end
+        @data_offset += FIELD_HEADER_SIZE + function_arg_buffer.bytesize
+        field_count += 4
+      end
+
+      max_records = 0
+      parts_full_size = 0
+      parts_partial_digest_size = 0
+      parts_partial_bval_size = 0
+
+      unless @node_partitions.nil?
+        parts_full_size = @node_partitions.parts_full.length * 2
+        parts_partial_digest_size = @node_partitions.parts_partial.length * 20
+
+        unless filter.nil?
+          parts_partial_bval_size = @node_partitions.parts_partial.length * 8
+        end
+        max_records = @node_partitions.record_max
+      end
+
+      if parts_full_size > 0
+        @data_offset += parts_full_size + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      if parts_partial_digest_size > 0
+        @data_offset += parts_partial_digest_size + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      if parts_partial_bval_size > 0
+        @data_offset += parts_partial_bval_size + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      # Estimate max records field size. This field is used in new servers and not used
+      # (but harmless to add) in old servers.
+      if max_records > 0
+        @data_offset += 8 + FIELD_HEADER_SIZE
+        field_count += 1
+      end
+
+      operation_count = 0
+      unless bin_names.empty?
+        # Estimate size for selected bin names (query bin names already handled for old servers).
+        bin_names.each do |bin_name|
+          estimate_operation_size_for_bin_name(bin_name)
+        end
+        operation_count = bin_names.length
+      end
+
+      projected_offset = @data_offset
+
+      size_buffer
+
+      read_attr = INFO1_READ
+      read_attr |= INFO1_NOBINDATA if !@policy.include_bin_data
+      read_attr |= INFO1_SHORT_QUERY if @policy.short_query
+
+      infoAttr = INFO3_PARTITION_DONE
+
+      write_header(@policy, read_attr, 0, field_count, operation_count)
+
+      write_field_string(@statement.namespace, FieldType::NAMESPACE) if @statement.namespace
+      write_field_string(@statement.set_name, FieldType::TABLE) if @statement.set_name
+
+      # Write records per second.
+      write_field_int(@policy.records_per_second, FieldType::RECORDS_PER_SECOND) if @policy.records_per_second > 0
+
+      write_filter_exp(@policy.filter_exp, exp_size)
+
+      # Write socket idle timeout.
+      write_field_int(@policy.socket_timeout, FieldType::SOCKET_TIMEOUT)
+
+      # Write task_id field
+      write_field_int64(@statement.task_id, FieldType::TRAN_ID)
+
+      unless predexp.nil?
+        write_field_header(pred_size, Aerospike::FieldType::PREDEXP)
+        @data_offset = Aerospike::PredExp.write(
+          predexp, @data_buffer, @data_offset
+        )
+      end
+
+      if filter
+        type = filter.collection_type
+
+        if type > 0
+          write_field_header(1, FieldType::INDEX_TYPE)
+          @data_offset += @data_buffer.write_byte(type, @data_offset)
+        end
+
+        write_field_header(filter_size, FieldType::INDEX_RANGE)
+        @data_offset += @data_buffer.write_byte(1, @data_offset)
+        @data_offset = filter.write(@data_buffer, @data_offset)
+
+        # TODO: Implement
+        # if packed_ctx
+        #   write_field_header(packed_ctx.length, FieldType::INDEX_CONTEXT)
+        #   @data_buffer.write_binary(packed_ctx, @data_offset)
+        # end
+      end
+
+      if @statement.function_name
+        write_field_header(1, FieldType::UDF_OP)
+        @data_offset += @data_buffer.write_byte(1, @data_offset)
+        write_field_string(@statement.package_name, FieldType::UDF_PACKAGE_NAME)
+        write_field_string(@statement.function_name, FieldType::UDF_FUNCTION)
+        write_field_string(function_arg_buffer, FieldType::UDF_ARGLIST)
+      end
+
+      if parts_full_size > 0
+        write_field_header(parts_full_size, FieldType::PID_ARRAY)
+        @node_partitions.parts_full.each do |part|
+          @data_offset += @data_buffer.write_uint16_little_endian(part.id, @data_offset)
+        end
+      end
+
+      if parts_partial_digest_size > 0
+        write_field_header(parts_partial_digest_size, FieldType::DIGEST_ARRAY)
+        @node_partitions.parts_partial.each do |part|
+          @data_offset += @data_buffer.write_binary(part.digest, @data_offset)
+        end
+      end
+
+      if parts_partial_bval_size > 0
+        write_field_header(parts_partial_bval_size, FieldType::BVAL_ARRAY)
+        @node_partitions.parts_partial.each do |part|
+          @data_offset += @data_buffer.write_uint64_little_endian(part.bval, @data_offset)
+        end
+      end
+
+      if max_records > 0
+        write_field(max_records, FieldType::MAX_RECORDS)
+      end
+
+      unless bin_names.empty?
+        bin_names.each do |bin_name|
+          write_operation_for_bin_name(bin_name, Operation::READ)
+        end
+      end
+
+      end_cmd
+
+      nil
+    end
+
+    def should_retry(e)
+      # !! converts nil to false
+      !!@tracker&.should_retry(@node_partitions, e)
+    end
+  end # class
+end # module

data/lib/aerospike/query/recordset.rb

@@ -22,7 +22,6 @@ module Aerospike
   # so the production and the consumptoin are decoupled
   # there can be an unlimited count of producer threads and consumer threads
   class Recordset
-
     attr_reader :records
 
     def initialize(queue_size = 5000, thread_count = 1, type)
@@ -66,18 +65,21 @@ module Aerospike
 
     # this is called by working threads to signal their job is finished
     # it decreases the count of active threads and puts an EOF on queue when all threads are finished
-    def thread_finished
+    # e is an exception that has happened in the exceutor, and outside of the threads themselves
+    def thread_finished(expn = nil)
       @active_threads.update do |v|
         v -= 1
         @records.enq(nil) if v == 0
         v
       end
+
+      raise expn unless expn.nil?
     end
 
     # this is called by a thread who faced an exception to singnal to terminate the whole operation
     # it also may be called by the user to terminate the command in the middle of fetching records from server nodes
     # it clears the queue so that if any threads are waiting for the queue get unblocked and find out about the cancellation
-    def cancel(expn=nil)
+    def cancel(expn = nil)
       set_exception(expn)
       @cancelled.set(true)
       @records.clear
@@ -104,18 +106,16 @@ module Aerospike
       @filters.nil? || @filters.empty?
     end
 
-  private
+    private
 
-    def set_exception(expn=nil)
+    def set_exception(expn = nil)
       expn ||= (@type == :scan ? SCAN_TERMINATED_EXCEPTION : QUERY_TERMINATED_EXCEPTION)
       @thread_exception.set(expn)
     end
-
   end
 
   private
 
-    SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
-    QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
-
+  SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
+  QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
 end

data/lib/aerospike/query/scan_executor.rb

@@ -17,26 +17,28 @@
 
 module Aerospike
   class ScanExecutor # :nodoc:
-
     def self.scan_partitions(policy, cluster, tracker, namespace, set_name, recordset, bin_names = nil)
       interval = policy.sleep_between_retries
 
       should_retry = false
 
       loop do
+        # reset last_expn
+        @last_expn = nil
+
         list = tracker.assign_partitions_to_nodes(cluster, namespace)
 
         if policy.concurrent_nodes
           threads = []
           # Use a thread per node
           list.each do |node_partition|
-
             threads << Thread.new do
               Thread.current.abort_on_exception = true
               command = ScanPartitionCommand.new(policy, tracker, node_partition, namespace, set_name, bin_names, recordset)
               begin
                 command.execute
               rescue => e
+                @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
                 should_retry ||= command.should_retry(e)
                 Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
               end
@@ -50,6 +52,7 @@ module Aerospike
             begin
               command.execute
             rescue => e
+              @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
               should_retry ||= command.should_retry(e)
               Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
             end
@@ -57,13 +60,12 @@ module Aerospike
         end
 
         if tracker.complete?(@cluster, policy) || !should_retry
-          recordset.thread_finished
+          recordset.thread_finished(@last_expn)
           return
         end
         sleep(interval) if policy.sleep_between_retries > 0
+        statement.reset_task_id
       end
     end
-
   end
-
 end

data/lib/aerospike/query/statement.rb

@@ -81,6 +81,13 @@ module Aerospike
     end
   end
 
+  def reset_task_id
+    @task_id = rand(RAND_MAX)
+    while @task_id == 0
+      @task_id = rand(RAND_MAX)
+    end
+  end
+
   private
 
   RAND_MAX = 2**63

data/lib/aerospike/query/stream_command.rb

@@ -80,7 +80,8 @@ module Aerospike
             raise expn
           end
 
-          @tracker&.set_last(@node_partitions, key, key.bval)
+          # UDF results do not return a key
+          @tracker&.set_last(@node_partitions, key, key.bval) if key
         end
       end # while