aerospike 2.9.1 → 2.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ec864bf7ffd197e9badf516904193eb4e6260ac257335facc8e3df058307384
-  data.tar.gz: '09fe60ac38789318a8739439d99490810129056614a479cafcecce01803c1bff'
+  metadata.gz: d0452d4d35ee6d5ce56586cc99b3a68c7e1b4a6c4f6334c6794414a483e2ef01
+  data.tar.gz: 328a5672510bb68e044a92c39fc1f04059daa53b4332eaa5a64dda04f5b3de78
 SHA512:
-  metadata.gz: 0cf87d41d1b4b04121bbf8589974bfa931dea4ec96f17bdf94db737797afb60dc05932f6f56eed0b8f138bb6684a45eb4974c489e878ed5a949209734bf2dae7
-  data.tar.gz: 65bccc5b55db802415541ae4b5307020477344bb2d771d7cc84b9554192fa97f76874c7538d2c62dbd0025780d91bd91c8dfc1e0143c2bd3753f092d1c391885
+  metadata.gz: 44c883336f3993d532c7989e1c6bf737deca2773cb444a4cfa6021bca35311c9d4fe5c74ddb73fa1274661435a127892f9a368bd1d94120e6965bee38b870216
+  data.tar.gz: ebac442b5b2ddc345b7d7713b0e8d7f0aaaba9da24aabe811e1a10a26555c2a85ba1d2bad0a0266674fdcc1c3deb573ed1839ffe98294dc979ccc69e42eab187

data/CHANGELOG.md

@@ -2,12 +2,58 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [2.14.0] - 2019-08-06
+
+* **New Features**
+  * Adds support for rake-aware reads.
+  * Adds support for client-server compression.
+
+* **Improvements**
+  * Adds support for `truncate-namespace` command.
+
+## [2.13.0] - 2019-07-17
+
+* **New Features**
+  * Adds support for replica policies.
+
+* **Improvements**
+  * Remove support for "old" partition tokenizer.
+  * Refactor how partition parser is initialized and called.
+  * Adds support for 'replicas' and remove the old partition table queries from the server.
+
+## [2.12.0] - 2019-04-21
+
+* **New Features**
+  * Support for predicate expressions in all transaction.
+  * Support for `operation.delete` in `client#operate`.
+
+* **Improvements**
+  * Optimize serialization for nested structures. Thanks to [@Kacper Madej](https://github.com/madejejej)! [[#94](https://github.com/aerospike/aerospike-client-ruby/pull/94)]
+  * Remove `Thread#abort_on_exception` from `batch_index_command`. Thanks to [@Kacper Madej](https://github.com/madejejej)! [[#94](https://github.com/aerospike/aerospike-client-ruby/pull/92)]
+  * Does not allow values other than Integer, Float, String, Symbol and nil to be used as keys in Maps.
+
+* **Bug Fixes**
+  * Fixes tests that weren't using ENV variables for connections. This will allow the tests to be run on any server.
+
+
+## [2.11.0] - 2019-05-17
+
+* **New Features**
+  * Support for predicate expressions in queries. Thanks to [@Minus10Degrees](https://github.com/Minus10Degrees)! [[#78](https://github.com/aerospike/aerospike-client-ruby/issues/78)]
+
+* **Bug Fixes**
+  * Client#execute\_udf\_on\_query should not modify the statement argument. [[#79](https://github.com/aerospike/aerospike-client-ruby/issues/79)]
+  * Encoding::UndefinedConversionError when reading blob data from CDT list/map bin. [[#84](https://github.com/aerospike/aerospike-client-ruby/issues/84)]
+
+## [2.10.0] - 2019-05-10
+
+* **New Features**
+  * Add support for LB discovery / seeding. Thanks to [@filiptepper](https://github.com/filiptepper)! [[#80](https://github.com/aerospike/aerospike-client-ruby/issues/80)]
 
 ## [2.9.1] - 2019-04-03
 
 * **Bug Fixes**
-  * Query fails if one or more cluster nodes do not have records in the set [[#77](https://github.com/aerospike/aerospike-client-ruby/issue/77)]
+  * Query fails if one or more cluster nodes do not have records in the set [[#77](https://github.com/aerospike/aerospike-client-ruby/issues/77)]
 
 * **Updates**
   * Change admin message version to 2 (from 0)
@@ -20,10 +66,10 @@ All notable changes to this project will be documented in this file.
   * Add INFINITY and WILDCARD values for use in CDT map/list comparators. [AER-5945]
 
 * **Bug Fixes**
-  * Default policies set on Client instance do not get applied [[#74](https://github.com/aerospike/aerospike-client-ruby/issue/74)]
+  * Default policies set on Client instance do not get applied [[#74](https://github.com/aerospike/aerospike-client-ruby/issues/74)]
 
 * **Updates**
-  * *BREAKING CHANGE*: Change default for send_key write policy to false [[#73](https://github.com/aerospike/aerospike-client-ruby/issue/73)]
+  * *BREAKING CHANGE*: Change default for send_key write policy to false [[#73](https://github.com/aerospike/aerospike-client-ruby/issues/73)]
   * Support truncate info command argument "lut=now" for servers that require it. [AER-5955]
 
 ## [2.8.0] - 2018-08-06

data/README.md

@@ -97,7 +97,7 @@ This library is packaged with a number of tests.
 
 To run all the test cases:
 
-    $ bundle exec rspec
+    $ AEROSPIKE_HOSTS="<host:port>[,<hoist:port>]" AEROSPIKE_USER="<user>" AEROSPIKE_PASSWORD="<pass>" bundle exec rspec
 
 <a name="Examples"></a>
 ## Examples

data/lib/aerospike.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -22,6 +22,7 @@ require "timeout"
 require 'resolv'
 require 'msgpack'
 require 'bcrypt'
+require 'zlib'
 
 require 'aerospike/atomic/atomic'
 
@@ -101,22 +102,25 @@ require 'aerospike/connection/create'
 
 require 'aerospike/cluster'
 require 'aerospike/cluster/create_connection'
-require 'aerospike/cluster/partition'
 require 'aerospike/cluster/find_nodes_to_remove'
 require 'aerospike/cluster/find_node'
-require 'aerospike/cluster/partition_tokenizer_new'
-require 'aerospike/cluster/partition_tokenizer_old'
+require 'aerospike/cluster/partition'
+require 'aerospike/cluster/partition_parser'
+require 'aerospike/cluster/rack_parser'
 require 'aerospike/node'
 require 'aerospike/node/generation'
+require 'aerospike/node/rebalance'
 require 'aerospike/node/refresh/failed'
 require 'aerospike/node/refresh/friends'
 require 'aerospike/node/refresh/info'
 require 'aerospike/node/refresh/partitions'
+require 'aerospike/node/refresh/racks'
 require 'aerospike/node/refresh/peers'
 require 'aerospike/node/refresh/reset'
 require 'aerospike/node/verify/cluster_name'
 require 'aerospike/node/verify/name'
 require 'aerospike/node/verify/partition_generation'
+require 'aerospike/node/verify/rebalance_generation'
 require 'aerospike/node/verify/peers_generation'
 require 'aerospike/node_validator'
 require 'aerospike/peer'
@@ -142,6 +146,15 @@ require 'aerospike/query/stream_command'
 require 'aerospike/query/query_command'
 require 'aerospike/query/scan_command'
 require 'aerospike/query/statement'
+require 'aerospike/query/pred_exp'
+
+require 'aerospike/query/pred_exp/and_or'
+require 'aerospike/query/pred_exp/geo_json_value'
+require 'aerospike/query/pred_exp/integer_value'
+require 'aerospike/query/pred_exp/op'
+require 'aerospike/query/pred_exp/regex'
+require 'aerospike/query/pred_exp/regex_flags'
+require 'aerospike/query/pred_exp/string_value'
 
 module Aerospike
   extend Loggable

data/lib/aerospike/aerospike_exception.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -90,5 +90,11 @@ module Aerospike
         super(ResultCode::COMMAND_REJECTED, msg)
       end
     end
+
+    class InvalidNamespace < Aerospike
+      def initialize(msg=nil)
+        super(ResultCode::INVALID_NAMESPACE, msg)
+      end
+    end
   end
 end

data/lib/aerospike/atomic/atomic.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/aerospike/bin.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2014-2017 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/list_operation.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_operation.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_order.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2017 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_policy.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_return_type.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2017 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/cdt/map_write_mode.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-# Copyright 2016-2018 Aerospike, Inc.
+# Copyright 2016-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.

data/lib/aerospike/client.rb

@@ -1,4 +1,4 @@
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -44,6 +44,7 @@ module Aerospike
     attr_accessor :default_read_policy
     attr_accessor :default_scan_policy
     attr_accessor :default_write_policy
+    attr_accessor :cluster
 
     def initialize(hosts = nil, policy: ClientPolicy.new, connect: true)
 
@@ -224,8 +225,19 @@ module Aerospike
     def truncate(namespace, set_name = nil, before_last_update = nil, options = {})
       policy = create_policy(options, Policy, default_info_policy)
 
-      str_cmd = "truncate:namespace=#{namespace}"
-      str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
+      node = @cluster.random_node
+      conn = node.get_connection(policy.timeout)
+
+      if set_name && !set_name.to_s.strip.empty?
+        str_cmd = "truncate:namespace=#{namespace}"
+        str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
+      else
+        if node.supports_feature(Aerospike::Features::TRUNCATE_NAMESPACE)
+          str_cmd = "truncate-namespace:namespace=#{namespace}"
+        else
+          str_cmd = "truncate:namespace=#{namespace}"
+        end
+      end
 
       if before_last_update
         lut_nanos = (before_last_update.to_f * 1_000_000_000.0).round
@@ -235,8 +247,7 @@ module Aerospike
         str_cmd << ";lut=now"
       end
 
-      # Send index command to one node. That node will distribute the command to other nodes.
-      response = send_info_command(policy, str_cmd).upcase
+      response = send_info_command(policy, str_cmd, node).upcase
       return if response == 'OK'
       raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_ERROR, "Truncate failed: #{response}")
     end
@@ -322,11 +333,11 @@ module Aerospike
 
       if policy.use_batch_direct
         key_map = BatchItem.generate_map(keys)
-        execute_batch_direct_commands(keys) do |node, batch|
+        execute_batch_direct_commands(policy, keys) do |node, batch|
           BatchDirectCommand.new(node, batch, policy, key_map, bin_names, results, info_flags)
         end
       else
-        execute_batch_index_commands(keys) do |node, batch|
+        execute_batch_index_commands(policy, keys) do |node, batch|
           BatchIndexCommand.new(node, batch, policy, bin_names, results, info_flags)
         end
       end
@@ -351,11 +362,11 @@ module Aerospike
 
       if policy.use_batch_direct
         key_map = BatchItem.generate_map(keys)
-        execute_batch_direct_commands(keys) do |node, batch|
+        execute_batch_direct_commands(policy, keys) do |node, batch|
           BatchDirectExistsCommand.new(node, batch, policy, key_map, results)
         end
       else
-        execute_batch_index_commands(keys) do |node, batch|
+        execute_batch_index_commands(policy, keys) do |node, batch|
           BatchIndexExistsCommand.new(node, batch, policy, results)
         end
       end
@@ -526,7 +537,7 @@ module Aerospike
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing UDF failed because cluster is empty.")
       end
 
-      # TODO: wait until all migrations are finished
+      statement = statement.clone
       statement.set_aggregate_function(package_name, function_name, function_args, false)
 
       # Use a thread per node
@@ -818,9 +829,13 @@ module Aerospike
       self.default_write_policy = create_policy(policies[:write], WritePolicy)
     end
 
-    def send_info_command(policy, command)
+    def send_info_command(policy, command, node = nil)
       Aerospike.logger.debug { "Sending info command: #{command}" }
-      _, response = @cluster.request_info(policy, command).first
+      if node
+        _, response = @cluster.request_node_info(node, policy, command).first
+      else
+        _, response = @cluster.request_info(policy, command).first
+      end
       response.to_s
     end
 
@@ -884,17 +899,16 @@ module Aerospike
       command.execute
     end
 
-    def execute_batch_index_commands(keys)
+    def execute_batch_index_commands(policy, keys)
       if @cluster.nodes.empty?
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing Batch Index command failed because cluster is empty.")
       end
 
-      batch_nodes = BatchIndexNode.generate_list(@cluster, keys)
+      batch_nodes = BatchIndexNode.generate_list(@cluster, policy.replica, keys)
       threads = []
 
       batch_nodes.each do |batch|
         threads << Thread.new do
-          Thread.current.abort_on_exception = true
           command = yield batch.node, batch
           execute_command(command)
         end
@@ -903,12 +917,12 @@ module Aerospike
       threads.each(&:join)
     end
 
-    def execute_batch_direct_commands(keys)
+    def execute_batch_direct_commands(policy, keys)
       if @cluster.nodes.empty?
         raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Executing Batch Direct command failed because cluster is empty.")
       end
 
-      batch_nodes = BatchDirectNode.generate_list(@cluster, keys)
+      batch_nodes = BatchDirectNode.generate_list(@cluster, policy.replica, keys)
       threads = []
 
       # Use a thread per namespace per node

data/lib/aerospike/cluster.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2014-2018 Aerospike, Inc.
+# Copyright 2014-2020 Aerospike, Inc.
 #
 # Portions may be licensed to Aerospike, Inc. under one or more contributor
 # license agreements.
@@ -28,6 +28,7 @@ module Aerospike
     attr_reader :features, :tls_options
     attr_reader :cluster_id, :aliases
     attr_reader :cluster_name
+    attr_accessor :rack_aware, :rack_id
 
     def initialize(policy, hosts)
       @cluster_seeds = hosts
@@ -37,6 +38,10 @@ module Aerospike
       @tend_interval = policy.tend_interval
       @cluster_name = policy.cluster_name
       @tls_options = policy.tls
+      @rack_aware = policy.rack_aware
+      @rack_id = policy.rack_id
+
+      @replica_index = Atomic.new(0)
 
       @aliases = {}
       @cluster_nodes = []
@@ -102,18 +107,128 @@ module Aerospike
       (node_array.length > 0) && !@closed.value
     end
 
-    def get_node_for_key(key)
-      partition = Partition.new_by_key(key)
+    # Returns a node on the cluster for read operations
+    def batch_read_node(partition, replica_policy)
+      case replica_policy
+        when Aerospike::Replica::MASTER, Aerospike::Replica::SEQUENCE
+          return master_node(partition)
+        when Aerospike::Replica::MASTER_PROLES
+          return master_proles_node(partition)
+        when Aerospike::Replica::PREFER_RACK
+          return rack_node(partition, seq)
+        when Aerospike::Replica::RANDOM
+          return random_node
+        else
+          raise Aerospike::Exceptions::InvalidNode("invalid policy.replica value")
+      end
+    end
+
+    # Returns a node on the cluster for read operations
+    def read_node(partition, replica_policy, seq)
+      case replica_policy
+        when Aerospike::Replica::MASTER
+          return master_node(partition)
+        when Aerospike::Replica::MASTER_PROLES
+          return master_proles_node(partition)
+        when Aerospike::Replica::PREFER_RACK
+          return rack_node(partition, seq)
+        when Aerospike::Replica::SEQUENCE
+          return sequence_node(partition, seq)
+        when Aerospike::Replica::RANDOM
+          return random_node
+        else
+          raise Aerospike::Exceptions::InvalidNode("invalid policy.replica value")
+      end
+    end
+
+    # Returns a node on the cluster for read operations
+    def master_node(partition)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      node_array = (replica_array.get)[0]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !node_array
+
+      node = (node_array.get)[partition.partition_id]
+      raise Aerospike::Exceptions::InvalidNode if !node || !node.active?
+
+      node
+    end
+
+    # Returns a node on the cluster
+    def rack_node(partition, seq)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      is_retry = seq.value > -1
+
+      node = nil
+      fallback = nil
+      for i in 1..replica_array.length
+        idx = (seq.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
+
+        next if !node
+
+        fallback = node
+
+        # If fallback exists, do not retry on node where command failed,
+        # even if fallback is not on the same rack.
+        return fallback if is_retry && fallback && i == replica_array.length
+
+        return node if node && node.active? && node.has_rack(partition.namespace, @rack_id)
+      end
+
+      return fallback if fallback
+
+      raise Aerospike::Exceptions::InvalidNode
+    end
 
-      # Must copy hashmap reference for copy on write semantics to work.
-      nmap = partitions
-      if node_array = nmap[partition.namespace]
-        node = node_array.value[partition.partition_id]
+    # Returns a node on the cluster for read operations
+    def master_proles_node(partition)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      node = nil
+      for replica in replica_array
+        idx = (@replica_index.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
+
+        return node if node && node.active?
+      end
+
+      raise Aerospike::Exceptions::InvalidNode
+    end
+
+    # Returns a random node on the cluster
+    def sequence_node(partition, seq)
+      partition_map = partitions
+      replica_array = partition_map[partition.namespace]
+      raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
+
+      replica_array = replica_array.get
+
+      node = nil
+      for replica in replica_array
+        idx = (seq.update{|v| v.succ} % replica_array.size).abs
+        node = (replica_array[idx].get)[partition.partition_id]
 
         return node if node && node.active?
       end
 
-      random_node
+      raise Aerospike::Exceptions::InvalidNode
+    end
+
+    def get_node_for_key(replica_policy, key)
+      partition = Partition.new_by_key(key)
+      batch_read_node(partition, replica_policy)
     end
 
     # Returns a random node on the cluster
@@ -124,8 +239,8 @@ module Aerospike
       i = 0
       while i < length
         # Must handle concurrency with other non-tending threads, so node_index is consistent.
-        index = (@node_index.update{ |v| v+1 } % node_array.length).abs
-        node = node_array[index]
+        idx = (@node_index.update{ |v| v.succ } % node_array.length).abs
+        node = node_array[idx]
 
         return node if node.active?
 
@@ -167,12 +282,9 @@ module Aerospike
       end
     end
 
-    def update_partitions(tokens, node)
-      nmap = tokens.update_partition(partitions, node)
-      # update partition write map
+    def update_partitions(parser)
+      nmap = parser.update_partitions(partitions)
       set_partitions(nmap) if nmap
-
-      Aerospike.logger.info("Partitions for node #{node.name} updated")
     end
 
     def request_info(policy, *commands)
@@ -183,6 +295,13 @@ module Aerospike
       end
     end
 
+    def request_node_info(node, policy, *commands)
+      conn = node.get_connection(policy.timeout)
+      Info.request(conn, *commands).tap do
+        node.put_connection(conn)
+      end
+    end
+
     def supports_feature?(feature)
       @features.get.include?(feature.to_s)
     end
@@ -279,6 +398,7 @@ module Aerospike
 
       nodes.each do |node|
         node.refresh_partitions(peers) if node.partition_generation.changed?
+        node.refresh_racks if node.rebalance_generation.changed?
       end
 
       if peers.generation_changed? || !peers.use_peers?
@@ -441,8 +561,10 @@ module Aerospike
     def find_node_in_partition_map(filter)
       partitions_list = partitions
 
-      partitions_list.values.each do |node_array|
-        return true if node_array.value.any? { |node| node == filter }
+      partitions_list.values.each do |replica_array|
+        replica_array.get.each do |node_array|
+          return true if node_array.value.any? { |node| node == filter }
+        end
       end
       false
     end