aerospike 0.1.3 → 0.1.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ee7d4805db6b56945a1f5112c62272f3371c79f8
+  data.tar.gz: ab922b1acac8a4aa5b09d5605886127c12af60cf
+SHA512:
+  metadata.gz: 6879a1edd54bcdb9ad82e5cc3f484cbe5f56034eed1058c9aeb556b2d7b1ca93f663f2260a7d1818b311d5ab65f11a5ab2b8fbae1b85bbaaf277fc5c01234df0
+  data.tar.gz: 911f4c56dfe6e1d6eb801e560f87732db0309dd5e80bec9969bd5f1af17f0ec3ecfdb249ea865178b619afa6eccbaaa956bc199eba2b9b2bce7d3bdafffe22a9

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## Dec 8 2014 (0.1.5)
+
+  Major features added, minor fixes and improvements.
+
+  * **New Features**:
+
+    * Added `Client.scan_node`, `Client.scan_all`
+    * Added `Client.query`
+
+  * **Fixes**
+
+    * Fixed getting back results only for specified bin names.
+
 ## Oct 27 2014 (0.1.3)
 
   Minor fix.

data/README.md

@@ -28,15 +28,15 @@ include Aerospike
 
 client = Client.new("127.0.0.1", 3000)
 
-key = Key.new('test', 'set name', 'key value')
-bins = {
+key = Key.new('test', 'test', 'key value')
+bin_map = {
   'bin1' => 'value1',
   'bin2' => 2,
   'bin4' => ['value4', {'map1' => 'map val'}],
   'bin5' => {'value5' => [124, "string value"]},
 }
 
-client.put(key, bins)
+client.put(key, bin_map)
 record = client.get(key)
 record.bins['bin1'] = 'other value'
 
@@ -83,6 +83,13 @@ Supported operating systems:
 5. Build and Install the gem locally: ```rake build && rake install```
 6. Run the benchmark: ```./tools/benchmark/benchmark.rb -u```
 
+<a name="Performance"></a>
+## Performance Tweaking
+
+We are bending all efforts to improve the client's performance. In out reference benchmarks, Go client performs almost as good as the C client.
+
+To read about performance variables, please refer to [`docs/performance.md`](docs/performance.md)
+
 <a name="Tests"></a>
 ## Tests
 
@@ -100,13 +107,13 @@ A variety of example applications are provided in the [`examples`](examples) dir
 See the [`examples/README.md`](examples/README.md) for details.
 
 <a name="Tools"></a>
-## Tools
+### Tools
 
 A variety of clones of original tools are provided in the [`tools`](tools) directory.
 They show how to use more advanced features of the library to reimplement the same functionality in a more concise way.
 
 <a name="Benchmarks"></a>
-#### Benchmarks
+## Benchmarks
 
 Benchmark utility is provided in the [`tools/benchmark`](tools/benchmark) directory.
 See the [`tools/benchmark/README.md`](tools/benchmark/README.md) for details.

data/lib/aerospike.rb

@@ -5,7 +5,8 @@ require "monitor"
 require "timeout"
 require 'resolv'
 require 'msgpack'
-require 'atomic'
+
+require 'aerospike/atomic/atomic'
 
 require 'aerospike/client'
 require 'aerospike/utils/pool'
@@ -36,12 +37,16 @@ require 'aerospike/command/read_command'
 require 'aerospike/command/delete_command'
 require 'aerospike/key'
 require 'aerospike/operation'
+
 require 'aerospike/policy/client_policy'
 require 'aerospike/policy/priority'
 require 'aerospike/policy/record_exists_action'
 require 'aerospike/policy/generation_policy'
 require 'aerospike/policy/policy'
 require 'aerospike/policy/write_policy'
+require 'aerospike/policy/scan_policy'
+require 'aerospike/policy/query_policy'
+
 require 'aerospike/cluster/connection'
 require 'aerospike/cluster/cluster'
 require 'aerospike/cluster/node_validator'
@@ -64,6 +69,12 @@ require 'aerospike/task/udf_register_task'
 require 'aerospike/task/task'
 require 'aerospike/language'
 
+require 'aerospike/query/recordset'
+require 'aerospike/query/filter'
+require 'aerospike/query/stream_command'
+require 'aerospike/query/query_command'
+require 'aerospike/query/scan_command'
+
 module Aerospike
   extend Loggable
 end

data/lib/aerospike/atomic/atomic.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+# Copyright 2014 Aerospike, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Aerospike
+
+  # Container object for client policy command.
+  class Atomic
+
+    def initialize(value)
+      @value = value
+
+      @mutex = Mutex.new
+    end
+
+    def update(&block)
+      @mutex.synchronize do
+        @value = block.call(@value)
+      end
+    end
+
+    def get
+      ret = nil
+      @mutex.synchronize do
+        ret = @value
+      end
+      ret
+    end
+    alias_method :value, :get
+
+    def set(value)
+      @mutex.synchronize do
+        @value = value
+      end
+    end
+    alias_method 'value='.to_sym, :set
+
+  end # class
+
+end # module

data/lib/aerospike/client.rb

@@ -41,6 +41,8 @@ module Aerospike
     def initialize(host, port, options={})
       @default_policy = Policy.new
       @default_write_policy = WritePolicy.new
+      @default_scan_policy = ScanPolicy.new
+      @default_query_policy = QueryPolicy.new
 
       policy = opt_to_client_policy(options)
 
@@ -402,13 +404,14 @@ module Aerospike
       str_cmd << "content-len=#{content.length};udf-type=#{language};"
       # Send UDF to one node. That node will distribute the UDF to other nodes.
       response_map = @cluster.request_info(@default_policy, str_cmd)
-      response, _ = response_map.first
 
       res = {}
-      vals = response.split(';')
-      vals.each do |pair|
-        k, v = pair.split("=", 2)
-        res[k] = v
+      response_map.each do |k, response|
+        vals = response.to_s.split(';')
+        vals.each do |pair|
+          k, v = pair.split("=", 2)
+          res[k] = v
+        end
       end
 
       if res['error']
@@ -500,7 +503,7 @@ module Aerospike
         raise Aerospike::Exceptions::Aerospike.new(UDF_BAD_RESPONSE, "#{obj}")
       end
 
-      raise Aerospike::Exception::Aerospike.new(UDF_BAD_RESPONSE, "Invalid UDF return value")
+      raise Aerospike::Exceptions::Aerospike.new(UDF_BAD_RESPONSE, "Invalid UDF return value")
     end
 
     #  Create secondary index.
@@ -560,6 +563,141 @@ module Aerospike
       @cluster.request_info(@default_policy, *commands)
     end
 
+    #-------------------------------------------------------
+    # Scan Operations
+    #-------------------------------------------------------
+
+    def scan_all(namespace, set_name, bin_names=[], options={})
+      policy = opt_to_scan_policy(options)
+
+      # wait until all migrations are finished
+      # TODO: implement
+      # @cluster.WaitUntillMigrationIsFinished(policy.timeout)
+
+      # Retry policy must be one-shot for scans.
+      # copy on write for policy
+      new_policy = policy.clone
+
+      nodes = @cluster.nodes
+      if nodes.length == 0
+        raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Scan failed because cluster is empty.")
+      end
+
+      recordset = Recordset.new(policy.record_queue_size, nodes.length, :scan)
+
+      if policy.concurrent_nodes
+        # Use a thread per node
+        nodes.each do |node|
+          Thread.new do
+            abort_on_exception = true
+            command = ScanCommand.new(node, new_policy, namespace, set_name, bin_names, recordset)
+            begin
+              command.execute
+            rescue => e
+              Aerospike.logger.error(e) unless e == Rescordset::SCAN_TERMINATED_EXCEPTION 
+              recordset.cancel(e)
+            ensure
+              recordset.thread_finished
+            end
+          end
+        end
+      else
+        Thread.new do
+          abort_on_exception = true
+          nodes.each do |node|
+            command = ScanCommand.new(node, new_policy, namespace, set_name, bin_names, recordset)
+            begin
+              command.execute
+            rescue => e
+              Aerospike.logger.error(e) unless e == Rescordset::SCAN_TERMINATED_EXCEPTION 
+              recordset.cancel(e)
+            ensure
+              recordset.thread_finished
+            end
+          end
+        end
+      end
+
+      recordset
+    end
+
+    # ScanNode reads all records in specified namespace and set, from one node only.
+    # The policy can be used to specify timeouts.
+    def scan_node(node, namespace, set_name, bin_names=[], options={})
+      policy = opt_to_scan_policy(options)
+      # wait until all migrations are finished
+      # TODO: implement
+      # @cluster.WaitUntillMigrationIsFinished(policy.timeout)
+
+      # Retry policy must be one-shot for scans.
+      # copy on write for policy
+      new_policy = policy.clone
+      new_policy.max_retries = 0
+
+      node = @cluster.get_node_by_name(node) if !node.is_a?(Aerospike::Node)
+
+      recordset = Recordset.new(policy.record_queue_size, 1, :scan)
+
+      Thread.new do
+        abort_on_exception = true
+        command = ScanCommand.new(node, new_policy, namespace, set_name, bin_names, recordset)
+        begin
+          command.execute
+        rescue => e
+          Aerospike.logger.error(e) unless e == Rescordset::SCAN_TERMINATED_EXCEPTION 
+          recordset.cancel(e)
+        ensure
+          recordset.thread_finished
+        end
+      end
+
+      recordset
+    end
+
+    #--------------------------------------------------------
+    # Query functions (Supported by Aerospike 3 servers only)
+    #--------------------------------------------------------
+
+    # Query executes a query and returns a recordset.
+    # The query executor puts records on a channel from separate goroutines.
+    # The caller can concurrently pops records off the channel through the
+    # record channel.
+    #
+    # This method is only supported by Aerospike 3 servers.
+    # If the policy is nil, a default policy will be generated.
+    def query(statement, options={})
+      policy = opt_to_query_policy(options)
+      new_policy = policy.clone
+
+      # Always set a taskId
+      statement.task_id = Time.now.to_i if statement.task_id == 0
+
+      nodes = @cluster.nodes
+      if nodes.length == 0
+        raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_NOT_AVAILABLE, "Scan failed because cluster is empty.")
+      end
+
+      recordset = Recordset.new(policy.record_queue_size, nodes.length, :query)
+
+      # Use a thread per node
+      nodes.each do |node|
+        Thread.new do
+          abort_on_exception = true
+          command = QueryCommand.new(node, new_policy, statement, recordset)
+          begin
+            command.execute
+          rescue => e
+            Aerospike.logger.error(e) unless e == Rescordset::QUERY_TERMINATED_EXCEPTION 
+            recordset.cancel(e)
+          ensure
+            recordset.thread_finished
+          end
+        end
+      end
+
+      recordset
+    end
+
     private
 
     def send_info_command(policy, command)
@@ -580,7 +718,7 @@ module Aerospike
     end
 
     def opt_to_client_policy(options)
-      if options == {} || options.nil?
+      if options.nil? || options == {}
         ClientPolicy.new
       elsif options.is_a?(ClientPolicy)
         options
@@ -594,7 +732,7 @@ module Aerospike
     end
 
     def opt_to_policy(options)
-      if options == {} || options.nil?
+      if options.nil? || options == {}
         @default_policy
       elsif options.is_a?(Policy)
         options
@@ -609,7 +747,7 @@ module Aerospike
     end
 
     def opt_to_write_policy(options)
-      if options == {} || options.nil?
+      if options.nil? || options == {}
         @default_write_policy
       elsif options.is_a?(WritePolicy)
         options
@@ -624,6 +762,31 @@ module Aerospike
       end
     end
 
+    def opt_to_scan_policy(options)
+      if options.nil? || options == {}
+        @default_scan_policy
+      elsif options.is_a?(ScanPolicy)
+        options
+      elsif options.is_a?(Hash)
+        ScanPolicy.new(
+          options[:scan_percent],
+          options[:concurrent_nodes],
+          options[:include_bin_data],
+          options[:fail_on_cluster_change]
+        )
+      end
+    end
+
+    def opt_to_query_policy(options)
+      if options.nil? || options == {}
+        @default_query_policy
+      elsif options.is_a?(QueryPolicy)
+        options
+      elsif options.is_a?(Hash)
+        QueryPolicy.new()
+      end
+    end
+
     def batch_execute(keys, &cmd_gen)
       batch_nodes = BatchNode.generate_list(@cluster, keys)
       threads = []

data/lib/aerospike/cluster/cluster.rb

@@ -17,7 +17,7 @@
 require 'thread'
 require 'timeout'
 
-require 'atomic'
+require 'aerospike/atomic/atomic'
 
 module Aerospike
 
@@ -210,7 +210,7 @@ module Aerospike
             refresh_count += 1
             friend_list.concat(friends) if friends
           rescue => e
-            Aerospike.logger.warn("Node `#{node}` refresh failed: #{e.to_s}")
+            Aerospike.logger.error("Node `#{node}` refresh failed: #{e.to_s}")
           end
         end
       end
@@ -285,7 +285,7 @@ module Aerospike
         begin
           seed_node_validator = NodeValidator.new(seed, @connection_timeout)
         rescue => e
-          Aerospike.logger.warn("Seed #{seed.to_s} failed: #{e}")
+          Aerospike.logger.error("Seed #{seed.to_s} failed: #{e}")
           next
         end
 
@@ -299,7 +299,7 @@ module Aerospike
             begin
               nv = NodeValidator.new(aliass, @connection_timeout)
             rescue Exection => e
-              Aerospike.logger.warn("Seed #{seed.to_s} failed: #{e}")
+              Aerospike.logger.error("Seed #{seed.to_s} failed: #{e}")
               next
             end
           end
@@ -368,7 +368,7 @@ module Aerospike
           list << node
 
         rescue => e
-          Aerospike.logger.warn("Add node #{node.to_s} failed: #{e}")
+          Aerospike.logger.error("Add node #{node.to_s} failed: #{e}")
         end
       end
 
@@ -398,7 +398,7 @@ module Aerospike
 
         when 2
           # Two node clusters require at least one successful refresh before removing.
-          if refresh_count == 1 && node.reference_count.value == 0 && !node.responded.value
+          if refresh_count == 2 && node.reference_count.value == 0 && !node.responded.value
             # Node is not referenced nor did it respond.
             remove_list << node
           end