aerospike 0.1.3 → 0.1.5

data/lib/aerospike/policy/query_policy.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+# Copyright 2014 Aerospike, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'aerospike/policy/batch_policy'
+
+module Aerospike
+
+  # Container object for scan policy command.
+  class QueryPolicy < BatchPolicy
+
+    def initialize()
+      super()
+
+      @max_retries = 0
+
+      self
+    end
+
+  end # class
+
+end # module

data/lib/aerospike/policy/scan_policy.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+# Copyright 2014 Aerospike, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'aerospike/policy/batch_policy'
+
+module Aerospike
+
+  # Container object for scan policy command.
+  class ScanPolicy < BatchPolicy
+
+    attr_accessor :scan_percent, :concurrent_nodes,
+      :include_bin_data, :fail_on_cluster_change
+
+    def initialize(scan_percent=nil, concurrent_nodes=nil, include_bin_data=nil, fail_on_cluster_change=nil)
+      super()
+
+      @scan_percent = scan_percent || 100
+      @concurrent_nodes = concurrent_nodes.nil? ? true : concurrent_nodes
+      @include_bin_data = include_bin_data.nil? ? true : include_bin_data
+      @fail_on_cluster_change = fail_on_cluster_change.nil? ? true : fail_on_cluster_change
+
+      @max_retries = 0
+
+      self
+    end
+
+  end # class
+
+end # module

data/lib/aerospike/query/filter.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+# Copyright 2014 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module Aerospike
+
+  class Filter
+
+    def self.Equal(bin_name, value)
+      Filter.new(bin_name, value, value)
+    end
+
+    def self.Range(bin_name, from, to)
+      Filter.new(bin_name, from, to)
+    end
+
+    def estimate_size
+      return @name.bytesize + @begin.estimate_size + @end.estimate_size + 10
+    end
+
+    def write(buf, offset)
+      # Write name.
+      len = buf.write_binary(@name, offset+1)
+      buf.write_byte(len, offset)
+      offset += len + 1
+
+      # Write particle type.
+      buf.write_byte(@begin.type, offset)
+      offset+=1
+
+      # Write filter begin.
+      len = @begin.write(buf, offset+4)
+      buf.write_int32(len, offset)
+      offset += len + 4
+
+      # Write filter end.
+      len = @end.write(buf, offset+4)
+      buf.write_int32(len, offset)
+      offset += len + 4
+
+      offset
+    end
+
+    private
+
+    def initialize(bin_name, begin_value, end_value)
+      @name = bin_name
+      @begin = Aerospike::Value.of(begin_value)
+      @end = Aerospike::Value.of(end_value)
+    end
+
+  end # class
+
+end

data/lib/aerospike/query/query_command.rb

@@ -0,0 +1,200 @@
+# encoding: utf-8
+# Copyright 2014 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'aerospike/query/stream_command'
+require 'aerospike/query/recordset'
+
+module Aerospike
+
+  private
+
+  class QueryCommand < StreamCommand
+
+    def initialize(node, policy, statement, recordset)
+      super(node)
+
+      @policy = policy
+      @statement = statement
+      @recordset = recordset
+    end
+
+    def write_buffer
+      fieldCount = 0
+      filterSize = 0
+      binNameSize = 0
+
+      begin_cmd
+
+      if @statement.namespace
+        @data_offset += @statement.namespace.bytesize + FIELD_HEADER_SIZE
+        fieldCount+=1
+      end
+
+      if @statement.index_name
+        @data_offset += @statement.index_name.bytesize + FIELD_HEADER_SIZE
+        fieldCount+=1
+      end
+
+      if @statement.set_name
+        @data_offset += @statement.set_name.bytesize + FIELD_HEADER_SIZE
+        fieldCount+=1
+      end
+
+      if @statement.filters && @statement.filters.length > 0
+        @data_offset += FIELD_HEADER_SIZE
+        filterSize+=1 # num filters
+
+        @statement.filters.each do |filter|
+          sz = filter.estimate_size
+          filterSize += sz
+        end
+        @data_offset += filterSize
+        fieldCount+=1
+      else
+        # Calling query with no filters is more efficiently handled by a primary index scan.
+        # Estimate scan options size.
+        @data_offset += (2 + FIELD_HEADER_SIZE)
+        fieldCount+=1
+      end
+
+      if @statement.bin_names && @statement.bin_names.length > 0
+        @data_offset += FIELD_HEADER_SIZE
+        binNameSize+=1 # num bin names
+
+        @statement.bin_names.each do |bin_name|
+          binNameSize += bin_name.bytesize + 1 #OPERATION_HEADER_SIZE
+        end
+        @data_offset += binNameSize
+        fieldCount+=1
+      end
+
+      @statement.task_id = Time.now.to_i + Time.usec if @statement.task_id == 0
+
+      @data_offset += 8 + FIELD_HEADER_SIZE
+      fieldCount+=1
+
+      if @statement.function_name
+        @data_offset += FIELD_HEADER_SIZE + 1 # udf type
+        @data_offset += @statement.package_name.bytesize + FIELD_HEADER_SIZE
+        @data_offset += @statement.function_name.bytesize + FIELD_HEADER_SIZE
+
+        if @statement.function_args && @statement.function_args.length > 0
+          functionArgBuffer = Value.of(@statement.function_args).bytes
+        else
+          functionArgBuffer = 0.ord
+        end
+        @data_offset += FIELD_HEADER_SIZE + functionArgBuffer.bytesize
+        fieldCount += 4
+      end
+
+      if @statement.filters.nil? || @statement.filters.empty?
+        if @statement.bin_names && @statement.bin_names.length > 0
+          @statement.bin_names.each do |bin_name|
+            estimate_operation_size_for_bin_name(bin_name)
+          end
+        end
+      end
+
+      size_buffer
+
+      readAttr = INFO1_READ
+      operation_count = (@statement.filters.to_a.length == 0 && @statement.bin_names.to_a.length == 0) ? @statement.bin_names.length : 0
+
+      write_header(readAttr, 0, fieldCount, operation_count)
+
+      if @statement.namespace
+        write_field_string(@statement.namespace, Aerospike::FieldType::NAMESPACE)
+      end
+
+      unless @statement.index_name.nil?
+        write_field_string(@statement.index_name, Aerospike::FieldType::INDEX_NAME)
+      end
+
+      if @statement.set_name
+        write_field_string(@statement.set_name, Aerospike::FieldType::TABLE)
+      end
+
+      if @statement.filters && @statement.filters.length > 0
+        write_field_header(filterSize, Aerospike::FieldType::INDEX_RANGE)
+        @data_buffer.write_byte(@statement.filters.length, @data_offset)
+        @data_offset+=1
+
+        @statement.filters.each do |filter|
+          @data_offset = filter.write(@data_buffer, @data_offset)
+        end
+
+        # Query bin names are specified as a field (Scan bin names are specified later as operations)
+        if @statement.bin_names && @statement.bin_names.length > 0
+          write_field_header(binNameSize, Aerospike::FieldType::QUERY_BINLIST)
+          @data_buffer.write_byte(@statement.bin_names.length, @data_offset)
+          @data_offset += 1
+
+          @statement.bin_names.each do |bin_name|
+            len = @data_buffer.write_binary(bin_name, @data_offset + 1)
+            @data_buffer.write_byte(len, @data_offset)
+            @data_offset += len + 1;
+          end
+        end
+      else
+        # Calling query with no filters is more efficiently handled by a primary index scan.
+        write_field_header(2, Aerospike::FieldType::SCAN_OPTIONS)
+        priority = @policy.priority.ord
+        priority = priority << 4
+        @data_buffer.write_byte(priority, @data_offset)
+        @data_offset+=1
+        @data_buffer.write_byte(100.ord, @data_offset)
+        @data_offset+=1
+      end
+
+      if @statement.bin_names && @statement.bin_names.length > 0
+        write_field_header(binNameSize, Aerospike::FieldType::QUERY_BINLIST)
+        @data_buffer.write_byte(@statement.bin_names.length, @data_offset)
+        @data_offset+=1
+
+        @statement.bin_names.each do |bin_name|
+          len = @data_buffer.write_binary(bin_name, @data_offset + 1)
+          @data_buffer.write_byte(len, @data_offset)
+          @data_offset += len + 1
+        end
+      end
+
+      write_field_header(8, Aerospike::FieldType::TRAN_ID)
+      @data_buffer.write_int64(@statement.task_id, @data_offset)
+      @data_offset += 8
+
+      if @statement.function_name
+        write_field_header(1, Aerospike::FieldType::UDF_OP)
+        if @statement.return_data
+          @data_buffer.write_byte(1, @data_offset)
+          @data_offset+=1
+        else
+          @data_buffer.write_byte(2, @data_offset)
+          @data_offset+=1
+        end
+
+        write_field_string(@statement.package_name, Aerospike::FieldType::UDF_PACKAGE_NAME)
+        write_field_string(@statement.function_name, Aerospike::FieldType::UDF_FUNCTION)
+        write_field_bytes(functionArgBuffer, Aerospike::FieldType::UDF_ARGLIST)
+      end
+
+      end_cmd
+
+      return nil
+    end
+
+  end # class
+
+end # module

data/lib/aerospike/query/recordset.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+# Copyright 2014 Aerospike, Inc.
+#
+# Portions may be licensed to Aerospike, Inc. under one or more contributor
+# license agreements.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module Aerospike
+
+  # Recordset implements a queue for a producer-consumer pattern
+  # a producer is a thread that fetches records from one node and puts them on this queue
+  # a consumer fetches records from this queue
+  # so the production and the consumptoin are decoupled
+  # there can be an unlimited count of producer threads and consumer threads 
+  class Recordset
+
+    attr_reader :records
+
+    def initialize(queue_size = 5000, thread_count = 1, type)
+      queue_size = thread_count if queue_size < thread_count
+      @records = SizedQueue.new(queue_size)
+
+      # holds the count of active threads. 
+      # when it reaches zero it means the whole operations of fetching records from server nodes is finished
+      @active_threads = Atomic.new(thread_count)
+
+      # operation cancelled by user or an exception occured in one of the threads
+      @cancelled = Atomic.new(false)
+
+      # saves the exception that occurred inside one of the threads to reraise it in the main thread
+      # and also is a signal to terminate other threads as the whole operation is assumed as failed
+      @thread_exception = Atomic.new(nil)
+
+      # type of the operation. it is either :scan or :query
+      @type = type
+    end
+
+    # fetches and return the first record from the queue
+    # if the operation is not finished and the queue is empty it blocks and waits for new records
+    # it sets the exception if it reaches the EOF mark, and returns nil
+    # EOF means the operation has finished and no more records are comming from server nodes
+    # it re-raises the exception occurred in threads, or which was set after reaching the EOF in the previous call 
+    def next_record
+      raise @thread_exception.get unless @thread_exception.get.nil?
+
+      r = @records.deq
+
+      set_exception if r.nil?
+
+      r
+    end
+
+    # recordset is active unless it is cancelled by the user or an exception has occurred in of threads
+    def active?
+      !@cancelled.get
+    end
+
+    # this is called by working threads to signal their job is finished
+    # it decreases the count of active threads and puts an EOF on queue when all threads are finished
+    def thread_finished
+      @active_threads.update do |v|
+        v -= 1
+        @records.enq(nil) if v == 0
+        v
+      end
+    end
+
+    # this is called by a thread who faced an exception to singnal to terminate the whole operation
+    # it also may be called by the user to terminate the command in the middle of fetching records from server nodes
+    # it clears the queue so that if any threads are waiting for the queue get unblocked and find out about the cancellation 
+    def cancel(expn=nil)
+      set_exception(expn)
+      @cancelled.set(true)
+      @records.clear
+    end
+
+    # fetches and returns all the records from the queue until the whole operation is finished and it reaches an EOF mark
+    # calling cancel inside the each block raises an exception to signal other consumer threads
+    def each(&block)
+      r = true
+      while r
+        r = next_record
+        # nil means EOF
+        unless r.nil?
+          block.call(r)
+        else
+          # reached the EOF
+          break
+        end
+      end
+    end
+
+    # the command is a scan if there are no filters applied otherwise it is a query
+    def is_scan?
+      @filters.nil? || @filters.empty?
+    end
+
+  private
+
+    def set_exception(expn=nil)
+      expn ||= (@type == :scan ? SCAN_TERMINATED_EXCEPTION : QUERY_TERMINATED_EXCEPTION)
+      @thread_exception.set(expn)
+    end
+
+  end
+
+  private
+
+    SCAN_TERMINATED_EXCEPTION = Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SCAN_TERMINATED)
+    QUERY_TERMINATED_EXCEPTION = Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SCAN_TERMINATED)
+
+end