aerospike 2.24.0 → 2.26.0

data/lib/aerospike/policy/policy.rb

@@ -13,26 +13,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'aerospike/policy/priority'
-require 'aerospike/policy/consistency_level'
-require 'aerospike/policy/replica'
-
+require "aerospike/policy/priority"
+require "aerospike/policy/consistency_level"
+require "aerospike/policy/replica"
 
 module Aerospike
 
   # Container object for client policy command.
   class Policy
-
-    attr_accessor :priority, :timeout, :max_retries, :sleep_between_retries, :consistency_level,
+    attr_accessor :filter_exp, :priority, :timeout, :max_retries, :sleep_between_retries, :consistency_level,
                   :predexp, :fail_on_filtered_out, :replica, :use_compression
 
     alias total_timeout timeout
     alias total_timeout= timeout=
 
-    def initialize(opt={})
+    def initialize(opt = {})
       # Container object for transaction policy attributes used in all database
       # operation calls.
 
+      # Optional expression filter. If filterExp exists and evaluates to false, the
+      # transaction is ignored.
+      #
+      # Default: nil
+      #
+      # ==== Examples:
+      #
+      # p = Policy.new
+      # p.filter_exp = Exp.build(Exp.eq(Exp.int_bin("a"), Exp.int_val(11)));
+      @filter_exp = opt[:filter_exp]
+
+      #  Throw exception if {#filter_exp} is defined and that filter evaluates
+      #  to false (transaction ignored).  The {AerospikeException}
+      #  will contain result code {ResultCode::FILTERED_OUT}.
+      #
+      #  This field is not applicable to batch, scan or query commands.
+      #
+      #  Default: false
+      @fail_on_filtered_out = opt[:fail_on_filtered_out] || false
+
       # Priority of request relative to other transactions.
       # Currently, only used for scans.
       @priority = opt[:priority] || Priority::DEFAULT
@@ -74,7 +92,6 @@ module Aerospike
       # ]
       @predexp = opt[:predexp] || nil
 
-
       # Throw exception if @predexp is defined and that filter evaluates
       # to false (transaction ignored). The Aerospike::Exceptions::Aerospike
       # will contain result code Aerospike::ResultCode::FILTERED_OUT.
@@ -86,7 +103,6 @@ module Aerospike
       # read operation.
       @consistency_level = opt[:consistency_level] || Aerospike::ConsistencyLevel::CONSISTENCY_ONE
 
-
       # Send read commands to the node containing the key's partition replica type.
       # Write commands are not affected by this setting, because all writes are directed
       # to the node containing the key's master partition.
@@ -118,8 +134,5 @@ module Aerospike
       # timeout was not exceeded. Enter zero to skip sleep.
       @sleep_between_retries = opt[:sleep_between_retries] || 0.5
     end
-
-
   end # class
-
 end # module

data/lib/aerospike/query/filter.rb

@@ -15,39 +15,51 @@
 # the License.
 
 module Aerospike
-
   class Filter
+    attr_reader :packed_ctx
 
-    def self.Equal(bin_name, value)
-      Filter.new(bin_name, value, value)
-    end
+    # open up the class to alias the class methods for naming consistency
+    class << self
+      def equal(bin_name, value, ctx: nil)
+        Filter.new(bin_name, value, value, nil, nil, ctx)
+      end
 
-    def self.Contains(bin_name, value, col_type)
-      Filter.new(bin_name, value, value, nil, col_type)
-    end
+      def contains(bin_name, value, col_type, ctx: nil)
+        Filter.new(bin_name, value, value, nil, col_type, ctx)
+      end
 
-    def self.Range(bin_name, from, to, col_type = nil)
-      Filter.new(bin_name, from, to, nil, col_type)
-    end
+      def range(bin_name, from, to, col_type = nil, ctx: nil)
+        Filter.new(bin_name, from, to, nil, col_type, ctx)
+      end
 
-    def self.geoWithinGeoJSONRegion(bin_name, region, col_type = nil)
-      region = region.to_json
-      Filter.new(bin_name, region, region, ParticleType::GEOJSON, col_type)
-    end
+      def geo_within_geo_region(bin_name, region, col_type = nil, ctx: nil)
+        region = region.to_json
+        Filter.new(bin_name, region, region, ParticleType::GEOJSON, col_type, ctx)
+      end
 
-    def self.geoWithinRadius(bin_name, lon, lat, radius_meter, col_type = nil)
-      region = GeoJSON.new({type: "AeroCircle", coordinates: [[lon, lat], radius_meter]})
-      geoWithinGeoJSONRegion(bin_name, region, col_type)
-    end
+      def geo_within_radius(bin_name, lon, lat, radius_meter, col_type = nil, ctx: nil)
+        region = GeoJSON.new({ type: "AeroCircle", coordinates: [[lon, lat], radius_meter] })
+        geo_within_geo_region(bin_name, region, col_type, ctx: ctx)
+      end
 
-    def self.geoContainsGeoJSONPoint(bin_name, point, col_type = nil)
-      point = point.to_json
-      Filter.new(bin_name, point, point, ParticleType::GEOJSON, col_type)
-    end
+      def geo_contains_geo_point(bin_name, point, col_type = nil, ctx: nil)
+        point = point.to_json
+        Filter.new(bin_name, point, point, ParticleType::GEOJSON, col_type, ctx)
+      end
 
-    def self.geoContainsPoint(bin_name, lon, lat, col_type = nil)
-      point = GeoJSON.new({type: "Point", coordinates: [lon, lat]})
-      geoContainsGeoJSONPoint(bin_name, point, col_type)
+      def geo_contains_point(bin_name, lon, lat, col_type = nil, ctx: nil)
+        point = GeoJSON.new({ type: "Point", coordinates: [lon, lat] })
+        geo_contains_geo_point(bin_name, point, col_type, ctx: ctx)
+      end
+
+      # alias the old names for compatibility
+      alias :Equal :equal
+      alias :Contains :contains
+      alias :Range :range
+      alias :geoWithinGeoJSONRegion :geo_within_geo_region
+      alias :geoWithinRadius :geo_within_radius
+      alias :geoContainsGeoJSONPoint :geo_contains_geo_point
+      alias :geoContainsPoint :geo_contains_point
     end
 
     def estimate_size
@@ -56,21 +68,21 @@ module Aerospike
 
     def write(buf, offset)
       # Write name.
-      len = buf.write_binary(@name, offset+1)
+      len = buf.write_binary(@name, offset + 1)
       buf.write_byte(len, offset)
       offset += len + 1
 
       # Write particle type.
       buf.write_byte(@val_type, offset)
-      offset+=1
+      offset += 1
 
       # Write filter begin.
-      len = @begin.write(buf, offset+4)
+      len = @begin.write(buf, offset + 4)
       buf.write_int32(len, offset)
       offset += len + 4
 
       # Write filter end.
-      len = @end.write(buf, offset+4)
+      len = @end.write(buf, offset + 4)
       buf.write_int32(len, offset)
       offset += len + 4
 
@@ -98,7 +110,7 @@ module Aerospike
 
     private
 
-    def initialize(bin_name, begin_value, end_value, val_type = nil, col_type = nil)
+    def initialize(bin_name, begin_value, end_value, val_type = nil, col_type = nil, ctx = nil)
       @name = bin_name
       @begin = Aerospike::Value.of(begin_value)
       @end = Aerospike::Value.of(end_value)
@@ -107,8 +119,8 @@ module Aerospike
       # but in certain cases caller can override the type.
       @val_type = val_type || @begin.type
       @col_type = col_type
-    end
 
+      @packed_ctx = CDT::Context.bytes(ctx)
+    end
   end # class
-
 end

data/lib/aerospike/query/query_executor.rb

@@ -17,28 +17,29 @@
 
 module Aerospike
   class QueryExecutor # :nodoc:
-
     def self.query_partitions(cluster, policy, tracker, statement, recordset)
       interval = policy.sleep_between_retries
 
       should_retry = false
 
       loop do
+        # reset last_expn
+        @last_expn = nil
+
         list = tracker.assign_partitions_to_nodes(cluster, statement.namespace)
 
         if policy.concurrent_nodes
           threads = []
           # Use a thread per node
           list.each do |node_partition|
-
             threads << Thread.new do
               Thread.current.abort_on_exception = true
               command = QueryPartitionCommand.new(node_partition.node, tracker, policy, statement, recordset, node_partition)
               begin
                 command.execute
               rescue => e
+                @last_expn = e unless e == QUERY_TERMINATED_EXCEPTION
                 should_retry ||= command.should_retry(e)
-                # puts "should retry: #{should_retry}"
                 Aerospike.logger.error(e.backtrace.join("\n")) unless e == QUERY_TERMINATED_EXCEPTION
               end
             end
@@ -51,23 +52,20 @@ module Aerospike
             begin
               command.execute
             rescue => e
+              @last_expn = e unless e == QUERY_TERMINATED_EXCEPTION
               should_retry ||= command.should_retry(e)
               Aerospike.logger.error(e.backtrace.join("\n")) unless e == QUERY_TERMINATED_EXCEPTION
             end
           end
         end
 
-        complete = tracker.complete?(@cluster, policy)
-
-        if complete || !should_retry
-          recordset.thread_finished
+        if tracker.complete?(@cluster, policy) || !should_retry
+          recordset.thread_finished(@last_expn)
           return
         end
         sleep(interval) if policy.sleep_between_retries > 0
         statement.reset_task_id
       end
     end
-
   end
-
 end

data/lib/aerospike/query/query_partition_command.rb

@@ -14,15 +14,13 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-require 'aerospike/query/stream_command'
-require 'aerospike/query/recordset'
+require "aerospike/query/stream_command"
+require "aerospike/query/recordset"
 
 module Aerospike
-
   private
 
   class QueryPartitionCommand < QueryCommand #:nodoc:
-
     def initialize(node, tracker, policy, statement, recordset, node_partitions)
       super(node, policy, statement, recordset, @node_partitions)
       @node_partitions = node_partitions
@@ -39,29 +37,29 @@ module Aerospike
 
       if @statement.namespace
         @data_offset += @statement.namespace.bytesize + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       if @statement.set_name
         @data_offset += @statement.set_name.bytesize + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       # Estimate recordsPerSecond field size. This field is used in new servers and not used
       # (but harmless to add) in old servers.
       if @policy.records_per_second > 0
         @data_offset += 4 + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       # Estimate socket timeout field size. This field is used in new servers and not used
       # (but harmless to add) in old servers.
       @data_offset += 4 + FIELD_HEADER_SIZE
-      field_count+=1
+      field_count += 1
 
       # Estimate task_id field.
       @data_offset += 8 + FIELD_HEADER_SIZE
-      field_count+=1
+      field_count += 1
 
       filter = @statement.filters[0]
       bin_names = @statement.bin_names
@@ -73,23 +71,22 @@ module Aerospike
         # Estimate INDEX_TYPE field.
         if col_type > 0
           @data_offset += FIELD_HEADER_SIZE + 1
-          field_count+=1
+          field_count += 1
         end
 
         # Estimate INDEX_RANGE field.
         @data_offset += FIELD_HEADER_SIZE
-        filter_size+=1  # num filters
+        filter_size += 1  # num filters
         filter_size += filter.estimate_size
 
         @data_offset += filter_size
-        field_count+=1
-
-        # TODO: Implement
-        # packed_ctx = filter.packed_ctx
-        # if packed_ctx
-        #   @data_offset += FIELD_HEADER_SIZE + packed_ctx.length
-        #   field_count+=1
-        # end
+        field_count += 1
+
+        packed_ctx = filter.packed_ctx
+        if packed_ctx
+          @data_offset += FIELD_HEADER_SIZE + packed_ctx.length
+          field_count += 1
+        end
       end
 
       @statement.set_task_id
@@ -102,13 +99,18 @@ module Aerospike
         field_count += 1
       end
 
+      unless @policy.filter_exp.nil?
+        exp_size = estimate_expression_size(@policy.filter_exp)
+        field_count += 1 if exp_size > 0
+      end
+
       # Estimate aggregation/background function size.
       if @statement.function_name
         @data_offset += FIELD_HEADER_SIZE + 1 # udf type
         @data_offset += @statement.package_name.bytesize + FIELD_HEADER_SIZE
         @data_offset += @statement.function_name.bytesize + FIELD_HEADER_SIZE
 
-        function_arg_buffer=''
+        function_arg_buffer = ""
         if @statement.function_args && @statement.function_args.length > 0
           function_arg_buffer = Value.of(@statement.function_args).to_bytes
         end
@@ -133,24 +135,24 @@ module Aerospike
 
       if parts_full_size > 0
         @data_offset += parts_full_size + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       if parts_partial_digest_size > 0
         @data_offset += parts_partial_digest_size + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       if parts_partial_bval_size > 0
         @data_offset += parts_partial_bval_size + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       # Estimate max records field size. This field is used in new servers and not used
       # (but harmless to add) in old servers.
       if max_records > 0
         @data_offset += 8 + FIELD_HEADER_SIZE
-        field_count+=1
+        field_count += 1
       end
 
       operation_count = 0
@@ -180,6 +182,8 @@ module Aerospike
       # Write records per second.
       write_field_int(@policy.records_per_second, FieldType::RECORDS_PER_SECOND) if @policy.records_per_second > 0
 
+      write_filter_exp(@policy.filter_exp, exp_size)
+
       # Write socket idle timeout.
       write_field_int(@policy.socket_timeout, FieldType::SOCKET_TIMEOUT)
 
@@ -205,11 +209,10 @@ module Aerospike
         @data_offset += @data_buffer.write_byte(1, @data_offset)
         @data_offset = filter.write(@data_buffer, @data_offset)
 
-        # TODO: Implement
-        # if packed_ctx
-        #   write_field_header(packed_ctx.length, FieldType::INDEX_CONTEXT)
-        #   @data_buffer.write_binary(packed_ctx, @data_offset)
-        # end
+        if packed_ctx
+          write_field_header(packed_ctx.length, FieldType::INDEX_CONTEXT)
+          @data_offset += @data_buffer.write_binary(packed_ctx, @data_offset)
+        end
       end
 
       if @statement.function_name
@@ -260,7 +263,5 @@ module Aerospike
       # !! converts nil to false
       !!@tracker&.should_retry(@node_partitions, e)
     end
-
   end # class
-
 end # module

data/lib/aerospike/query/recordset.rb

@@ -22,7 +22,6 @@ module Aerospike
   # so the production and the consumptoin are decoupled
   # there can be an unlimited count of producer threads and consumer threads
   class Recordset
-
     attr_reader :records
 
     def initialize(queue_size = 5000, thread_count = 1, type)
@@ -66,18 +65,21 @@ module Aerospike
 
     # this is called by working threads to signal their job is finished
     # it decreases the count of active threads and puts an EOF on queue when all threads are finished
-    def thread_finished
+    # e is an exception that has happened in the exceutor, and outside of the threads themselves
+    def thread_finished(expn = nil)
       @active_threads.update do |v|
         v -= 1
         @records.enq(nil) if v == 0
         v
       end
+
+      raise expn unless expn.nil?
     end
 
     # this is called by a thread who faced an exception to singnal to terminate the whole operation
     # it also may be called by the user to terminate the command in the middle of fetching records from server nodes
     # it clears the queue so that if any threads are waiting for the queue get unblocked and find out about the cancellation
-    def cancel(expn=nil)
+    def cancel(expn = nil)
       set_exception(expn)
       @cancelled.set(true)
       @records.clear
@@ -104,18 +106,16 @@ module Aerospike
       @filters.nil? || @filters.empty?
     end
 
-  private
+    private
 
-    def set_exception(expn=nil)
+    def set_exception(expn = nil)
       expn ||= (@type == :scan ? SCAN_TERMINATED_EXCEPTION : QUERY_TERMINATED_EXCEPTION)
       @thread_exception.set(expn)
     end
-
   end
 
   private
 
-    SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
-    QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
-
+  SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::ScanTerminated.new()
+  QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::QueryTerminated.new()
 end

data/lib/aerospike/query/scan_executor.rb

@@ -17,26 +17,28 @@
 
 module Aerospike
   class ScanExecutor # :nodoc:
-
     def self.scan_partitions(policy, cluster, tracker, namespace, set_name, recordset, bin_names = nil)
       interval = policy.sleep_between_retries
 
       should_retry = false
 
       loop do
+        # reset last_expn
+        @last_expn = nil
+
         list = tracker.assign_partitions_to_nodes(cluster, namespace)
 
         if policy.concurrent_nodes
           threads = []
           # Use a thread per node
           list.each do |node_partition|
-
             threads << Thread.new do
               Thread.current.abort_on_exception = true
               command = ScanPartitionCommand.new(policy, tracker, node_partition, namespace, set_name, bin_names, recordset)
               begin
                 command.execute
               rescue => e
+                @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
                 should_retry ||= command.should_retry(e)
                 Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
               end
@@ -50,6 +52,7 @@ module Aerospike
             begin
               command.execute
             rescue => e
+              @last_expn = e unless e == SCAN_TERMINATED_EXCEPTION
               should_retry ||= command.should_retry(e)
               Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
             end
@@ -57,13 +60,12 @@ module Aerospike
         end
 
         if tracker.complete?(@cluster, policy) || !should_retry
-          recordset.thread_finished
+          recordset.thread_finished(@last_expn)
           return
         end
         sleep(interval) if policy.sleep_between_retries > 0
+        statement.reset_task_id
       end
     end
-
   end
-
 end

data/lib/aerospike/task/execute_task.rb

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 module Aerospike
-
   private
 
   # ExecuteTask is used to poll for long running server execute job completion.
@@ -29,19 +28,24 @@ module Aerospike
       self
     end
 
-    # IsDone queries all nodes for task completion status.
+    # queries all nodes for task completion status.
     def all_nodes_done?
-
-      if @scan
-        command = 'scan-list'
-      else
-        command = 'query-list'
-      end
+      modul = @scan ? "scan" : "query"
+      cmd1 = "query-show:trid=#{@task_id}"
+      cmd2 = modul + "-show:trid=#{@task_id}"
+      cmd3 = "jobs:module=" + modul + ";cmd=get-job;trid=#{@task_id}"
 
       nodes = @cluster.nodes
       done = false
 
       nodes.each do |node|
+        command = cmd3
+        if node.supports_feature?(Aerospike::Features::PARTITION_QUERY)
+          command = cmd1
+        elsif node.supports_feature?(Aerospike::Features::QUERY_SHOW)
+          command = cmd2
+        end
+
         conn = node.get_connection(0)
         responseMap, _ = Info.request(conn, command)
         node.put_connection(conn)
@@ -58,28 +62,27 @@ module Aerospike
 
         b = index + find.length
         response = response[b, response.length]
-        find = 'job_status='
+        find = "job_status="
         index = response.index(find)
 
         next unless index
 
         b = index + find.length
         response = response[b, response.length]
-        e = response.index(':')
+        e = response.index(":")
         status = response[0, e]
 
         case status
-        when 'ABORTED'
+        when "ABORTED"
           raise Aerospike::Exceptions::QueryTerminated
-        when 'IN PROGRESS'
+        when "IN PROGRESS"
           return false
-        when 'DONE'
+        when "DONE"
           done = true
         end
       end
 
       done
     end
-
   end
 end