aerospike 2.24.0 → 2.25.0

data/lib/aerospike/policy/policy.rb

@@ -13,26 +13,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'aerospike/policy/priority'
-require 'aerospike/policy/consistency_level'
-require 'aerospike/policy/replica'
-
+require "aerospike/policy/priority"
+require "aerospike/policy/consistency_level"
+require "aerospike/policy/replica"
 
 module Aerospike
 
   # Container object for client policy command.
   class Policy
-
-    attr_accessor :priority, :timeout, :max_retries, :sleep_between_retries, :consistency_level,
+    attr_accessor :filter_exp, :priority, :timeout, :max_retries, :sleep_between_retries, :consistency_level,
                   :predexp, :fail_on_filtered_out, :replica, :use_compression
 
     alias total_timeout timeout
     alias total_timeout= timeout=
 
-    def initialize(opt={})
+    def initialize(opt = {})
       # Container object for transaction policy attributes used in all database
       # operation calls.
 
+      # Optional expression filter. If filterExp exists and evaluates to false, the
+      # transaction is ignored.
+      #
+      # Default: nil
+      #
+      # ==== Examples:
+      #
+      # p = Policy.new
+      # p.filter_exp = Exp.build(Exp.eq(Exp.int_bin("a"), Exp.int_val(11)));
+      @filter_exp = opt[:filter_exp]
+
+      #  Throw exception if {#filter_exp} is defined and that filter evaluates
+      #  to false (transaction ignored).  The {AerospikeException}
+      #  will contain result code {ResultCode::FILTERED_OUT}.
+      #
+      #  This field is not applicable to batch, scan or query commands.
+      #
+      #  Default: false
+      @fail_on_filtered_out = opt[:fail_on_filtered_out] || false
+
       # Priority of request relative to other transactions.
       # Currently, only used for scans.
       @priority = opt[:priority] || Priority::DEFAULT
@@ -74,7 +92,6 @@ module Aerospike
       # ]
       @predexp = opt[:predexp] || nil
 
-
       # Throw exception if @predexp is defined and that filter evaluates
       # to false (transaction ignored). The Aerospike::Exceptions::Aerospike
       # will contain result code Aerospike::ResultCode::FILTERED_OUT.
@@ -86,7 +103,6 @@ module Aerospike
       # read operation.
       @consistency_level = opt[:consistency_level] || Aerospike::ConsistencyLevel::CONSISTENCY_ONE
 
-
       # Send read commands to the node containing the key's partition replica type.
       # Write commands are not affected by this setting, because all writes are directed
       # to the node containing the key's master partition.
@@ -118,8 +134,5 @@ module Aerospike
       # timeout was not exceeded. Enter zero to skip sleep.
       @sleep_between_retries = opt[:sleep_between_retries] || 0.5
     end
-
-
   end # class
-
 end # module

data/lib/aerospike/query/query_executor.rb

@@ -17,28 +17,29 @@
 
 module Aerospike
   class QueryExecutor # :nodoc:
-
     def self.query_partitions(cluster, policy, tracker, statement, recordset)
       interval = policy.sleep_between_retries
 
       should_retry = false
 
       loop do
+        # reset last_expn
+        @last_expn = nil
+
         list = tracker.assign_partitions_to_nodes(cluster, statement.namespace)
 
         if policy.concurrent_nodes
           threads = []
           # Use a thread per node
           list.each do |node_partition|
-
             threads << Thread.new do
               Thread.current.abort_on_exception = true
               command = QueryPartitionCommand.new(node_partition.node, tracker, policy, statement, recordset, node_partition)
               begin
                 command.execute
               rescue => e
+                @last_expn = e unless e == QUERY_TERMINATED_EXCEPTION
                 should_retry ||= command.should_retry(e)
-                # puts "should retry: #{should_retry}"
                 Aerospike.logger.error(e.backtrace.join("\n")) unless e == QUERY_TERMINATED_EXCEPTION
               end
             end
@@ -51,23 +52,20 @@ module Aerospike
             begin
               command.execute
             rescue => e
+              @last_expn = e unless e == QUERY_TERMINATED_EXCEPTION
               should_retry ||= command.should_retry(e)
               Aerospike.logger.error(e.backtrace.join("\n")) unless e == QUERY_TERMINATED_EXCEPTION
             end
           end
         end
 
-        complete = tracker.complete?(@cluster, policy)
-
-        if complete || !should_retry
-          recordset.thread_finished
+        if tracker.complete?(@cluster, policy) || !should_retry
+          recordset.thread_finished(@last_expn)
           return
         end
         sleep(interval) if policy.sleep_between_retries > 0
         statement.reset_task_id
       end
     end
-
   end
-
 end

data/lib/aerospike/query/query_partition_command.rb

@@ -14,15 +14,13 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-require 'aerospike/query/stream_command'
-require 'aerospike/query/recordset'
+require "aerospike/query/stream_command"
+require "aerospike/query/recordset"
 
 module Aerospike
-
   private
 
   class QueryPartitionCommand < QueryCommand #:nodoc:
-
     def initialize(node, tracker, policy, statement, recordset, node_partitions)
       super(node, policy, statement, recordset, @node_partitions)
       @node_partitions = node_partitions
@@ -39,29 +37,29 @@ module Aerospike
 
       if @statement.namespace
         @data_offset += @statement.namespace.bytesize + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       if @statement.set_name
         @data_offset += @statement.set_name.bytesize + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       # Estimate recordsPerSecond field size. This field is used in new servers and not used
       # (but harmless to add) in old servers.
       if @policy.records_per_second > 0
         @data_offset += 4 + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       # Estimate socket timeout field size. This field is used in new servers and not used
       # (but harmless to add) in old servers.
       @data_offset += 4 + FIELD_HEADER_SIZE
-      field_count+=1
+      field_count += 1
 
       # Estimate task_id field.
       @data_offset += 8 + FIELD_HEADER_SIZE
-      field_count+=1
+      field_count += 1
 
       filter = @statement.filters[0]
       bin_names = @statement.bin_names
@@ -73,16 +71,16 @@ module Aerospike
         # Estimate INDEX_TYPE field.
         if col_type > 0
           @data_offset += FIELD_HEADER_SIZE + 1
-          field_count+=1
+          field_count += 1
         end
 
         # Estimate INDEX_RANGE field.
         @data_offset += FIELD_HEADER_SIZE
-        filter_size+=1  # num filters
+        filter_size += 1  # num filters
         filter_size += filter.estimate_size
 
         @data_offset += filter_size
-        field_count+=1
+        field_count += 1
 
         # TODO: Implement
         # packed_ctx = filter.packed_ctx
@@ -102,13 +100,18 @@ module Aerospike
         field_count += 1
       end
 
+      unless @policy.filter_exp.nil?
+        exp_size = estimate_expression_size(@policy.filter_exp)
+        field_count += 1 if exp_size > 0
+      end
+
       # Estimate aggregation/background function size.
       if @statement.function_name
         @data_offset += FIELD_HEADER_SIZE + 1 # udf type
         @data_offset += @statement.package_name.bytesize + FIELD_HEADER_SIZE
         @data_offset += @statement.function_name.bytesize + FIELD_HEADER_SIZE
 
-        function_arg_buffer=''
+        function_arg_buffer = ""
         if @statement.function_args && @statement.function_args.length > 0
           function_arg_buffer = Value.of(@statement.function_args).to_bytes
         end
@@ -133,24 +136,24 @@ module Aerospike
 
       if parts_full_size > 0
         @data_offset += parts_full_size + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       if parts_partial_digest_size > 0
         @data_offset += parts_partial_digest_size + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       if parts_partial_bval_size > 0
         @data_offset += parts_partial_bval_size + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       # Estimate max records field size. This field is used in new servers and not used
       # (but harmless to add) in old servers.
       if max_records > 0
         @data_offset += 8 + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       operation_count = 0
@@ -180,6 +183,8 @@ module Aerospike
       # Write records per second.
       write_field_int(@policy.records_per_second, FieldType::RECORDS_PER_SECOND) if @policy.records_per_second > 0
 
+      write_filter_exp(@policy.filter_exp, exp_size)
+
       # Write socket idle timeout.
       write_field_int(@policy.socket_timeout, FieldType::SOCKET_TIMEOUT)
 
@@ -260,7 +265,5 @@ module Aerospike
       # !! converts nil to false
       !!@tracker&.should_retry(@node_partitions, e)
     end
-
   end # class
-
 end # module

data/lib/aerospike/query/recordset.rb

@@ -22,7 +22,6 @@ module Aerospike
   # so the production and the consumptoin are decoupled
   # there can be an unlimited count of producer threads and consumer threads
   class Recordset
-
     attr_reader :records
 
     def initialize(queue_size = 5000, thread_count = 1, type)
@@ -66,18 +65,21 @@ module Aerospike
 
     # this is called by working threads to signal their job is finished
     # it decreases the count of active threads and puts an EOF on queue when all threads are finished
-    def thread_finished
+    # e is an exception that has happened in the exceutor, and outside of the threads themselves
+    def thread_finished(expn = nil)
       @active_threads.update do |v|
         v -= 1
         @records.enq(nil) if v == 0
         v
       end
+
+      raise expn unless expn.nil?
     end
 
     # this is called by a thread who faced an exception to singnal to terminate the whole operation
     # it also may be called by the user to terminate the command in the middle of fetching records from server nodes
     # it clears the queue so that if any threads are waiting for the queue get unblocked and find out about the cancellation
-    def cancel(expn=nil)
+    def cancel(expn = nil)
       set_exception(expn)
       @cancelled.set(true)
       @records.clear
@@ -104,18 +106,16 @@ module Aerospike
       @filters.nil? || @filters.empty?
     end
 
-  private
+    private
 
-    def set_exception(expn=nil)
+    def set_exception(expn = nil)
       expn ||= (@type == :scan ? SCAN_TERMINATED_EXCEPTION : QUERY_TERMINATED_EXCEPTION)
       @thread_exception.set(expn)
     end
-
   end
 
   private
 
-    SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
-    QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
-
+  SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
+  QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
 end

data/lib/aerospike/query/scan_executor.rb

@@ -17,26 +17,28 @@
 
 module Aerospike
   class ScanExecutor # :nodoc:
-
     def self.scan_partitions(policy, cluster, tracker, namespace, set_name, recordset, bin_names = nil)
       interval = policy.sleep_between_retries
 
       should_retry = false
 
       loop do
+        # reset last_expn
+        @last_expn = nil
+
         list = tracker.assign_partitions_to_nodes(cluster, namespace)
 
         if policy.concurrent_nodes
           threads = []
           # Use a thread per node
           list.each do |node_partition|
-
             threads << Thread.new do
               Thread.current.abort_on_exception = true
               command = ScanPartitionCommand.new(policy, tracker, node_partition, namespace, set_name, bin_names, recordset)
               begin
                 command.execute
               rescue => e
+                @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
                 should_retry ||= command.should_retry(e)
                 Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
               end
@@ -50,6 +52,7 @@ module Aerospike
             begin
               command.execute
             rescue => e
+              @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
               should_retry ||= command.should_retry(e)
               Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
             end
@@ -57,13 +60,12 @@ module Aerospike
         end
 
         if tracker.complete?(@cluster, policy) || !should_retry
-          recordset.thread_finished
+          recordset.thread_finished(@last_expn)
           return
         end
         sleep(interval) if policy.sleep_between_retries > 0
+        statement.reset_task_id
       end
     end
-
   end
-
 end

data/lib/aerospike/task/execute_task.rb

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 module Aerospike
-
   private
 
   # ExecuteTask is used to poll for long running server execute job completion.
@@ -29,19 +28,24 @@ module Aerospike
       self
     end
 
-    # IsDone queries all nodes for task completion status.
+    # queries all nodes for task completion status.
     def all_nodes_done?
-
-      if @scan
-        command = 'scan-list'
-      else
-        command = 'query-list'
-      end
+      modul = @scan ? "scan" : "query"
+      cmd1 = "query-show:trid=#{@task_id}"
+      cmd2 = modul + "-show:trid=#{@task_id}"
+      cmd3 = "jobs:module=" + modul + ";cmd=get-job;trid=#{@task_id}"
 
       nodes = @cluster.nodes
       done = false
 
       nodes.each do |node|
+        command = cmd3
+        if node.supports_feature?(Aerospike::Features::PARTITION_QUERY)
+          command = cmd1
+        elsif node.supports_feature?(Aerospike::Features::QUERY_SHOW)
+          command = cmd2
+        end
+
         conn = node.get_connection(0)
         responseMap, _ = Info.request(conn, command)
         node.put_connection(conn)
@@ -58,28 +62,27 @@ module Aerospike
 
         b = index + find.length
         response = response[b, response.length]
-        find = 'job_status='
+        find = "job_status="
         index = response.index(find)
 
         next unless index
 
         b = index + find.length
         response = response[b, response.length]
-        e = response.index(':')
+        e = response.index(":")
         status = response[0, e]
 
         case status
-        when 'ABORTED'
+        when "ABORTED"
           raise Aerospike::Exceptions::QueryTerminated
-        when 'IN PROGRESS'
+        when "IN PROGRESS"
           return false
-        when 'DONE'
+        when "DONE"
           done = true
         end
       end
 
       done
     end
-
   end
 end

data/lib/aerospike/utils/buffer.rb

@@ -17,36 +17,34 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-require 'aerospike/utils/pool'
+require "aerospike/utils/pool"
 
 module Aerospike
-
   private
 
   # Buffer class to ease the work around
   class Buffer #:nodoc:
-
     @@buf_pool = Pool.new
     @@buf_pool.create_proc = Proc.new { Buffer.new }
 
     attr_accessor :buf
 
-    INT16 = 's>'
-    UINT16 = 'n'
-    UINT16LE = 'v'
-    INT32 = 'l>'
-    UINT32 = 'N'
-    INT64 = 'q>'
-    UINT64 = 'Q>'
-    UINT64LE = 'Q'
-    DOUBLE = 'G'
+    INT16 = "s>"
+    UINT16 = "n"
+    UINT16LE = "v"
+    INT32 = "l>"
+    UINT32 = "N"
+    INT64 = "q>"
+    UINT64 = "Q>"
+    UINT64LE = "Q"
+    DOUBLE = "G"
 
     DEFAULT_BUFFER_SIZE = 16 * 1024
     MAX_BUFFER_SIZE = 10 * 1024 * 1024
 
-    def initialize(size=DEFAULT_BUFFER_SIZE, buf = nil)
+    def initialize(size = DEFAULT_BUFFER_SIZE, buf = nil)
       @buf = (buf ? buf : ("%0#{size}d" % 0))
-      @buf.force_encoding('binary')
+      @buf.force_encoding("binary")
       @slice_end = @buf.bytesize
     end
 
@@ -61,6 +59,7 @@ module Aerospike
     def size
       @buf.bytesize
     end
+
     alias_method :length, :size
 
     def eat!(n)
@@ -135,7 +134,7 @@ module Aerospike
       8
     end
 
-    def read(offset, len=nil)
+    def read(offset, len = nil)
       if len
         @buf[offset, len]
       else
@@ -144,37 +143,37 @@ module Aerospike
     end
 
     def read_int16(offset)
-      vals = @buf[offset..offset+1]
+      vals = @buf[offset..offset + 1]
       vals.unpack(INT16)[0]
     end
 
     def read_uint16(offset)
-      vals = @buf[offset..offset+1]
+      vals = @buf[offset..offset + 1]
       vals.unpack(UINT16)[0]
     end
 
     def read_int32(offset)
-      vals = @buf[offset..offset+3]
+      vals = @buf[offset..offset + 3]
       vals.unpack(INT32)[0]
     end
 
     def read_uint32(offset)
-      vals = @buf[offset..offset+3]
+      vals = @buf[offset..offset + 3]
       vals.unpack(UINT32)[0]
     end
 
     def read_int64(offset)
-      vals = @buf[offset..offset+7]
+      vals = @buf[offset..offset + 7]
       vals.unpack(INT64)[0]
     end
 
     def read_uint64_little_endian(offset)
-      vals = @buf[offset..offset+7]
+      vals = @buf[offset..offset + 7]
       vals.unpack(UINT64LE)[0]
     end
 
     def read_uint64(offset)
-      vals = @buf[offset..offset+7]
+      vals = @buf[offset..offset + 7]
       vals.unpack(UINT64)[0]
     end
 
@@ -183,14 +182,14 @@ module Aerospike
       i = 0
       while i < len
         val <<= 8
-        val |= @buf[offset+i].ord & 0xFF
+        val |= @buf[offset + i].ord & 0xFF
         i = i.succ
       end
       val
     end
 
     def read_double(offset)
-      vals = @buf[offset..offset+7]
+      vals = @buf[offset..offset + 7]
       vals.unpack(DOUBLE)[0]
     end
 
@@ -199,39 +198,48 @@ module Aerospike
     end
 
     def to_s
-      @buf[0..@slice_end-1]
+      @buf[0..@slice_end - 1]
     end
 
     def reset
-      for i in 0..@buf.size-1
-        @buf[i] = ' '
+      for i in 0..@buf.size - 1
+        @buf[i] = " "
       end
     end
 
-    def dump(start=0, finish=nil)
+    def dump(start = 0, finish = nil)
+      buf ||= @buf.bytes
       finish ||= @slice_end - 1
       width = 16
 
-      ascii = '|'
+      ascii = "|"
       counter = 0
 
-      print '%06x  ' % start
+      print "%08x  " % start
       @buf.bytes[start...finish].each do |c|
         if counter >= start
-          print '%02x ' % c
+          print "%02x " % c
           ascii << (c.between?(32, 126) ? c : ?.)
-          if ascii.length >= width
-            ascii << '|'
+          print " " if ascii.length == (width / 2 + 1)
+          if ascii.length > width
+            ascii << "|"
             puts ascii
-            ascii = '|'
-            print '%06x  ' % (counter + 1)
+            ascii = "|"
+            print "%08x  " % (counter + 1)
           end
         end
         counter += 1
       end
-      puts
-    end
 
+      # print the remainder in buffer
+      if ascii.length.positive?
+        fill_size = ((width - ascii.length + 1) * 3)
+        fill_size += 1 if ascii.length <= (width / 2)
+        filler = " " * fill_size
+        print filler
+        ascii << "|"
+        puts ascii
+      end
+    end
   end # buffer
-
 end # module

data/lib/aerospike/utils/packer.rb

@@ -14,13 +14,11 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-require 'msgpack'
-require 'aerospike/utils/pool'
+require "msgpack"
+require "aerospike/utils/pool"
 
 module Aerospike
-
   class Packer < MessagePack::Packer #:nodoc:
-
     AS_EXT_TYPE = -1
 
     @@pool = Pool.new
@@ -44,9 +42,12 @@ module Aerospike
       buffer << [val].pack("S>")
     end
 
+    def write_raw(buf)
+      buffer.write(buf)
+    end
+
     def bytes
-      self.to_s.force_encoding('binary')
+      self.to_s.force_encoding("binary")
     end
   end
-
 end