dynamoid 1.3.4 → 2.2.0

data/Rakefile

@@ -18,30 +18,6 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = FileList["spec/**/*_spec.rb"]
 end
 
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.pattern = "spec/**/*_spec.rb"
-  spec.rcov = true
-end
-
-desc "Start DynamoDBLocal, run tests, clean up"
-task :unattended_spec do |t|
-
-  if system("bin/start_dynamodblocal")
-    puts "DynamoDBLocal started; proceeding with specs."
-  else
-    raise "Unable to start DynamoDBLocal.  Cannot run unattended specs."
-  end
-
-  #Cleanup
-  at_exit do
-    unless system("bin/stop_dynamodblocal")
-      $stderr.puts "Unable to cleanly stop DynamoDBLocal."
-    end
-  end
-
-  Rake::Task["spec"].invoke
-end
-
 require "yard"
 YARD::Rake::YardocTask.new do |t|
   t.files   = ["lib/**/*.rb", "README", "LICENSE"]   # optional

data/Vagrantfile

@@ -18,7 +18,7 @@ Vagrant.configure('2') do |config|
     # Pillars
     salt.pillar({
       'ruby' => {
-        'version' => '2.3.3',
+        'version' => '2.4.1',
       }
     })
 

data/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '2'
+
+services:
+  dynamodb:
+    image: deangiberson/aws-dynamodb-local
+    ports:
+      - 8000:8000

data/dynamoid.gemspec

@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
       "Sumanth Ravipati",
       "Pascal Corpet",
       "Brian Glusman",
-      "Peter Boling"
+      "Peter Boling",
+      "Andrew Konchin"
   ]
   spec.email = ["peter.boling@gmail.com", "brian@stellaservice.com"]
 
@@ -50,13 +51,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency(%q<activesupport>, [">= 4"])
   spec.add_runtime_dependency(%q<aws-sdk-resources>, ["~> 2"])
   spec.add_runtime_dependency(%q<concurrent-ruby>, [">= 1.0"])
-  spec.add_development_dependency "pry", "~> 0.10"
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "appraisal", "~> 2.1"
   spec.add_development_dependency "wwtd", "~> 1.3"
   spec.add_development_dependency(%q<yard>, [">= 0"])
-  spec.add_development_dependency(%q<coveralls>, [">= 0"])
-  spec.add_development_dependency(%q<rspec-retry>, [">= 0"])
+  spec.add_development_dependency "coveralls", "~> 0.8"
 end

data/gemfiles/rails_4_0.gemfile

@@ -2,8 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "pry-byebug", platforms: :ruby
 gem "rails", "~> 4.0.0"
-# Still tested against Ruby 2.0.0, which can't install nokogiri 1.7+
-gem "nokogiri", "~> 1.6.8.1"
+gem "nokogiri", "~> 1.6.8"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4_1.gemfile

@@ -2,8 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "pry-byebug", platforms: :ruby
 gem "rails", "~> 4.1.0"
-# Still tested against Ruby 2.0.0, which can't install nokogiri 1.7+
-gem "nokogiri", "~> 1.6.8.1"
+gem "nokogiri", "~> 1.6.8"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4_2.gemfile

@@ -2,8 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "pry-byebug", platforms: :ruby
 gem "rails", "~> 4.2.0"
-# Still tested against Ruby 2.0.0, which can't install nokogiri 1.7+
-gem "nokogiri", "~> 1.6.8.1"
+gem "nokogiri", "~> 1.6.8"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_5_0.gemfile

@@ -2,6 +2,7 @@
 
 source "https://rubygems.org"
 
+gem "pry-byebug", platforms: :ruby
 gem "rails", "~> 5.0.0"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_5_1.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "pry-byebug", platforms: :ruby
+gem "rails", "~> 5.1.0"
+
+gemspec path: "../"

data/gemfiles/rails_5_2.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "pry-byebug", platforms: :ruby
+gem "rails", "~> 5.2.0"
+
+gemspec path: "../"

data/lib/dynamoid.rb

@@ -1,36 +1,36 @@
-require "delegate"
-require "time"
-require "securerandom"
-require "active_support"
-require "active_support/core_ext"
-require "active_support/json"
-require "active_support/inflector"
-require "active_support/lazy_load_hooks"
-require "active_support/time_with_zone"
-require "active_model"
-
-require "dynamoid/version"
-require "dynamoid/errors"
-require "dynamoid/fields"
-require "dynamoid/indexes"
-require "dynamoid/associations"
-require "dynamoid/persistence"
-require "dynamoid/dirty"
-require "dynamoid/validations"
-require "dynamoid/criteria"
-require "dynamoid/finders"
-require "dynamoid/identity_map"
-require "dynamoid/config"
-require "dynamoid/components"
-require "dynamoid/document"
-require "dynamoid/adapter"
-
-require "dynamoid/tasks/database"
-
-require "dynamoid/middleware/identity_map"
+require 'delegate'
+require 'time'
+require 'securerandom'
+require 'active_support'
+require 'active_support/core_ext'
+require 'active_support/json'
+require 'active_support/inflector'
+require 'active_support/lazy_load_hooks'
+require 'active_support/time_with_zone'
+require 'active_model'
+
+require 'dynamoid/version'
+require 'dynamoid/errors'
+require 'dynamoid/fields'
+require 'dynamoid/indexes'
+require 'dynamoid/associations'
+require 'dynamoid/persistence'
+require 'dynamoid/dirty'
+require 'dynamoid/validations'
+require 'dynamoid/criteria'
+require 'dynamoid/finders'
+require 'dynamoid/identity_map'
+require 'dynamoid/config'
+require 'dynamoid/components'
+require 'dynamoid/document'
+require 'dynamoid/adapter'
+
+require 'dynamoid/tasks/database'
+
+require 'dynamoid/middleware/identity_map'
 
 if defined?(Rails)
-  require "dynamoid/railtie"
+  require 'dynamoid/railtie'
 end
 
 module Dynamoid

data/lib/dynamoid/adapter.rb

@@ -51,7 +51,7 @@ module Dynamoid
     def benchmark(method, *args)
       start = Time.now
       result = yield
-      Dynamoid.logger.info "(#{((Time.now - start) * 1000.0).round(2)} ms) #{method.to_s.split('_').collect(&:upcase).join(' ')}#{ " - #{args.inspect}" unless args.nil? || args.empty? }"
+      Dynamoid.logger.debug "(#{((Time.now - start) * 1000.0).round(2)} ms) #{method.to_s.split('_').collect(&:upcase).join(' ')}#{ " - #{args.inspect}" unless args.nil? || args.empty? }"
       return result
     end
 
@@ -80,12 +80,12 @@ module Dynamoid
     #                        unless multiple ids are passed in.
     #
     # @since 0.2.0
-    def read(table, ids, options = {})
+    def read(table, ids, options = {}, &blk)
       range_key = options.delete(:range_key)
 
       if ids.respond_to?(:each)
         ids = ids.collect{|id| range_key ? [id, range_key] : id}
-        batch_get_item({table => ids}, options)
+        batch_get_item({table => ids}, options, &blk)
       else
         options[:range_key] = range_key if range_key
         get_item(table, ids, options)
@@ -99,13 +99,13 @@ module Dynamoid
     # @param [Array] range_key of the record to delete, can also be a string of just one range_key
     #
     def delete(table, ids, options = {})
-      range_key = options[:range_key] #array of range keys that matches the ids passed in
+      range_key = options[:range_key] # array of range keys that matches the ids passed in
       if ids.respond_to?(:each)
         if range_key.respond_to?(:each)
-          #turn ids into array of arrays each element being hash_key, range_key
-          ids = ids.each_with_index.map{|id,i| [id,range_key[i]]}
+          # turn ids into array of arrays each element being hash_key, range_key
+          ids = ids.each_with_index.map{|id, i| [id, range_key[i]]}
         else
-          ids = range_key ? [[ids, range_key]] : ids
+          ids = range_key ? ids.map { |id| [id, range_key] } : ids
         end
 
         batch_delete_item(table => ids)
@@ -120,7 +120,7 @@ module Dynamoid
     # @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan
     #
     # @since 0.2.0
-    def scan(table, query, opts = {})
+    def scan(table, query = {}, opts = {})
       benchmark('Scan', table, query) {adapter.scan(table, query, opts)}
     end
 
@@ -144,8 +144,12 @@ module Dynamoid
       # Method delegation with benchmark to the underlying adapter. Faster than relying on method_missing.
       #
       # @since 0.2.0
-      define_method(m) do |*args|
-        benchmark("#{m.to_s}", *args) {adapter.send(m, *args)}
+      define_method(m) do |*args, &blk|
+        if blk.present?
+          benchmark("#{m.to_s}", *args) { adapter.send(m, *args, &blk) }
+        else
+          benchmark("#{m.to_s}", *args) { adapter.send(m, *args) }
+        end
       end
     end
 

data/lib/dynamoid/adapter_plugin/aws_sdk_v2.rb

@@ -3,7 +3,7 @@ module Dynamoid
 
     # The AwsSdkV2 adapter provides support for the aws-sdk version 2 for ruby.
     class AwsSdkV2
-      EQ = "EQ".freeze
+      EQ = 'EQ'.freeze
       RANGE_MAP = {
           range_greater_than: 'GT',
           range_less_than:    'LT',
@@ -18,6 +18,7 @@ module Dynamoid
       # we declare schema in models
       FIELD_MAP = {
           eq:           'EQ',
+          ne:           'NE',
           gt:           'GT',
           lt:           'LT',
           gte:          'GE',
@@ -28,16 +29,16 @@ module Dynamoid
           contains:     'CONTAINS',
           not_contains: 'NOT_CONTAINS'
       }
-      HASH_KEY  = "HASH".freeze
-      RANGE_KEY = "RANGE".freeze
-      STRING_TYPE  = "S".freeze
-      NUM_TYPE     = "N".freeze
-      BINARY_TYPE  = "B".freeze
+      HASH_KEY  = 'HASH'.freeze
+      RANGE_KEY = 'RANGE'.freeze
+      STRING_TYPE  = 'S'.freeze
+      NUM_TYPE     = 'N'.freeze
+      BINARY_TYPE  = 'B'.freeze
       TABLE_STATUSES = {
-          creating: "CREATING",
-          updating: "UPDATING",
-          deleting: "DELETING",
-          active: "ACTIVE"
+          creating: 'CREATING',
+          updating: 'UPDATING',
+          deleting: 'DELETING',
+          active: 'ACTIVE'
       }.freeze
       PARSE_TABLE_STATUS = ->(resp, lookup = :table) {
         # lookup is table for describe_table API
@@ -45,6 +46,8 @@ module Dynamoid
         #   because Amazon, damnit.
         resp.send(lookup).table_status
       }
+      BATCH_WRITE_ITEM_REQUESTS_LIMIT = 25
+
       attr_reader :table_cache
 
       # Establish the connection to DynamoDB.
@@ -81,30 +84,55 @@ module Dynamoid
         @client
       end
 
-      # Puts or deletes multiple items in one or more tables
+      # Puts multiple items in one table
+      #
+      # If optional block is passed it will be called for each written batch of items, meaning once per batch.
+      # Block receives boolean flag which is true if there are some unprocessed items, otherwise false.
+      #
+      # @example Saves several items to the table testtable
+      #   Dynamoid::AdapterPlugin::AwsSdkV2.batch_write_item('table1', [{ id: '1', name: 'a' }, { id: '2', name: 'b'}])
+      #
+      # @example Pass block
+      #   Dynamoid::AdapterPlugin::AwsSdkV2.batch_write_item('table1', items) do |bool|
+      #     if bool
+      #       puts 'there are unprocessed items'
+      #     end
+      #   end
       #
       # @param [String] table_name the name of the table
       # @param [Array]  items to be processed
       # @param [Hash]   additional options
+      # @param [Proc]   optional block
       #
-      #See: http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
+      # See:
+      # * http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
+      # * http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
       def batch_write_item table_name, objects, options = {}
-        request_items = []
-        options ||= {}
-        objects.each do |o|
-          request_items << { "put_request" => { item: o } }
-        end
+        items = objects.map { |o| sanitize_item(o) }
 
         begin
-          client.batch_write_item(
-            {
-              request_items: {
-                table_name => request_items,
-              },
-              return_consumed_capacity: "TOTAL",
-              return_item_collection_metrics: "SIZE"
-            }.merge!(options)
-          )
+          while items.present? do
+            batch = items.shift(BATCH_WRITE_ITEM_REQUESTS_LIMIT)
+            requests = batch.map { |item| { put_request: { item: item } } }
+
+            response = client.batch_write_item(
+              {
+                request_items: {
+                  table_name => requests,
+                },
+                return_consumed_capacity: 'TOTAL',
+                return_item_collection_metrics: 'SIZE'
+              }.merge!(options)
+            )
+
+            if block_given?
+              yield(response.unprocessed_items.present?)
+            end
+
+            if response.unprocessed_items.present?
+              items += response.unprocessed_items[table_name].map { |r| r.put_request.item }
+            end
+          end
         rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
           raise Dynamoid::Errors::ConditionalCheckFailedException, e
         end
@@ -112,17 +140,37 @@ module Dynamoid
 
       # Get many items at once from DynamoDB. More efficient than getting each item individually.
       #
+      # If optional block is passed `nil` will be returned and the block will be called for each read batch of items,
+      # meaning once per batch.
+      #
+      # Block receives parameters:
+      # * hash with items like `{ table_name: [items]}`
+      # * and boolean flag is true if there are some unprocessed keys, otherwise false.
+      #
       # @example Retrieve IDs 1 and 2 from the table testtable
-      #   Dynamoid::AdapterPlugin::AwsSdkV2.batch_get_item({'table1' => ['1', '2']})
+      #   Dynamoid::AdapterPlugin::AwsSdkV2.batch_get_item('table1' => ['1', '2'])
+      #
+      # @example Pass block to receive each batch
+      #   Dynamoid::AdapterPlugin::AwsSdkV2.batch_get_item('table1' => ids) do |hash, bool|
+      #     puts hash['table1']
+      #
+      #     if bool
+      #       puts 'there are unprocessed keys'
+      #     end
+      #   end
       #
       # @param [Hash] table_ids the hash of tables and IDs to retrieve
       # @param [Hash] options to be passed to underlying BatchGet call
+      # @param [Proc] optional block can be passed to handle each batch of items
       #
       # @return [Hash] a hash where keys are the table names and the values are the retrieved items
       #
+      #  See:
+      #  * http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
+      #
       # @since 1.0.0
       #
-      # @todo: Provide support for passing options to underlying batch_get_item http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
+      # @todo: Provide support for passing options to underlying batch_get_item
       def batch_get_item(table_ids, options = {})
         request_items = Hash.new{|h, k| h[k] = []}
         return request_items if table_ids.all?{|k, v| v.blank?}
@@ -131,19 +179,22 @@ module Dynamoid
 
         table_ids.each do |t, ids|
           next if ids.blank?
+          ids = Array(ids).dup
           tbl = describe_table(t)
           hk  = tbl.hash_key.to_s
           rng = tbl.range_key.to_s
 
-          Array(ids).each_slice(Dynamoid::Config.batch_size) do |ids|
+          while ids.present? do
+            batch = ids.shift(Dynamoid::Config.batch_size)
+
             request_items = Hash.new{|h, k| h[k] = []}
 
             keys = if rng.present?
-              Array(ids).map do |h,r|
+              Array(batch).map do |h, r|
                 { hk => h, rng => r }
               end
             else
-              Array(ids).map do |id|
+              Array(batch).map do |id|
                 { hk => id }
               end
             end
@@ -156,35 +207,70 @@ module Dynamoid
               request_items: request_items
             )
 
-            results.data[:responses].each do |table, rows|
-              ret[table] += rows.collect { |r| result_item_to_hash(r) }
+            unless block_given?
+              results.data[:responses].each do |table, rows|
+                ret[table] += rows.collect { |r| result_item_to_hash(r) }
+              end
+            else
+              batch_results = Hash.new([].freeze)
+
+              results.data[:responses].each do |table, rows|
+                batch_results[table] += rows.collect { |r| result_item_to_hash(r) }
+              end
+
+              yield(batch_results, results.unprocessed_keys.present?)
+            end
+
+            if results.unprocessed_keys.present?
+              ids += results.unprocessed_keys[t].keys.map { |h| h[hk] }
             end
           end
         end
 
-        ret
+        unless block_given?
+          ret
+        end
       end
 
       # Delete many items at once from DynamoDB. More efficient than delete each item individually.
       #
       # @example Delete IDs 1 and 2 from the table testtable
       #   Dynamoid::AdapterPlugin::AwsSdk.batch_delete_item('table1' => ['1', '2'])
-      #or
+      # or
       #   Dynamoid::AdapterPlugin::AwsSdkV2.batch_delete_item('table1' => [['hk1', 'rk2'], ['hk1', 'rk2']]]))
       #
       # @param [Hash] options the hash of tables and IDs to delete
       #
-      # @return nil
+      # See:
+      # * http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
+      # * http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
       #
-      # @todo: Provide support for passing options to underlying delete_item http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#delete_item-instance_method
+      # TODO handle rejections because of internal processing failures
       def batch_delete_item(options)
+        requests = []
+
         options.each_pair do |table_name, ids|
           table = describe_table(table_name)
-          ids.each do |id|
-            client.delete_item(table_name: table_name, key: key_stanza(table, *id))
+
+          ids.each_slice(BATCH_WRITE_ITEM_REQUESTS_LIMIT) do |sliced_ids|
+            delete_requests = sliced_ids.map { |id|
+              {delete_request: {key: key_stanza(table, *id)}}
+            }
+
+            requests << {table_name => delete_requests}
           end
         end
-        nil
+
+        begin
+          requests.map do |request_items|
+            client.batch_write_item(
+              request_items: request_items,
+              return_consumed_capacity: 'TOTAL',
+              return_item_collection_metrics: 'SIZE')
+          end
+        rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
+          raise Dynamoid::Errors::ConditionalCheckFailedException, e
+        end
       end
 
       # Create a table on DynamoDB. This usually takes a long time to complete.
@@ -210,8 +296,8 @@ module Dynamoid
         gs_indexes = options[:global_secondary_indexes]
 
         key_schema = {
-          :hash_key_schema => { key => (options[:hash_key_type] || :string) },
-          :range_key_schema => options[:range_key]
+          hash_key_schema: { key => (options[:hash_key_type] || :string) },
+          range_key_schema: options[:range_key]
         }
         attribute_definitions = build_all_attribute_definitions(
           key_schema,
@@ -367,7 +453,7 @@ module Dynamoid
             key: key_stanza(table, key, range_key),
             attribute_updates: iu.to_h,
             expected: expected_stanza(conditions),
-            return_values: "ALL_NEW"
+            return_values: 'ALL_NEW'
           )
           result_item_to_hash(result[:attributes])
         rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
@@ -393,13 +479,8 @@ module Dynamoid
       #
       # See: http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#put_item-instance_method
       def put_item(table_name, object, options = {})
-        item = {}
         options ||= {}
-
-        object.each do |k, v|
-          next if v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
-          item[k.to_s] = v
-        end
+        item = sanitize_item(object)
 
         begin
           client.put_item(
@@ -452,52 +533,34 @@ module Dynamoid
         record_limit = opts.delete(:record_limit)
         scan_limit = opts.delete(:scan_limit)
         batch_size = opts.delete(:batch_size)
-        limit = [record_limit, scan_limit, batch_size].compact.min
-        q[:limit] = limit if limit
-
-        opts.delete(:next_token).tap do |token|
-          break unless token
-          q[:exclusive_start_key] = {
-            hk  => token[:hash_key_element],
-            rng => token[:range_key_element]
-          }
-          # For secondary indices the start key must contain the indices composite key
-          # but also the table's composite keys
-          q[:exclusive_start_key][table.hash_key] = token[:table_hash_key_element] if token[:table_hash_key_element]
-          q[:exclusive_start_key][table.range_key] = token[:table_range_key_element] if token[:table_range_key_element]
-        end
+        exclusive_start_key = opts.delete(:exclusive_start_key)
+        limit = [record_limit, scan_limit, batch_size].compact.min        
 
         key_conditions = {
           hk => {
-            # TODO: Provide option for other operators like NE, IN, LE, etc
             comparison_operator: EQ,
-            attribute_value_list: [
-              opts.delete(:hash_value).freeze
-            ]
+            attribute_value_list: attribute_value_list(EQ, opts.delete(:hash_value).freeze)
           }
         }
 
         opts.each_pair do |k, v|
-          # TODO: ATM, only few comparison operators are supported, provide support for all operators
           next unless(op = RANGE_MAP[k])
           key_conditions[rng] = {
             comparison_operator: op,
-            attribute_value_list: [
-              opts.delete(k).freeze
-            ].flatten # Flatten as BETWEEN operator specifies array of two elements
+            attribute_value_list: attribute_value_list(op, opts.delete(k).freeze)
           }
         end
 
         query_filter = {}
-        opts.reject {|k,_| k.in? RANGE_MAP.keys}.each do |attr, hash|
+        opts.reject {|k, _| k.in? RANGE_MAP.keys}.each do |attr, hash|
           query_filter[attr] = {
             comparison_operator: FIELD_MAP[hash.keys[0]],
-            attribute_value_list: [
-              hash.values[0].freeze
-            ].flatten # Flatten as BETWEEN operator specifies array of two elements
+            attribute_value_list: attribute_value_list(FIELD_MAP[hash.keys[0]], hash.values[0].freeze)
           }
         end
 
+        q[:limit] = limit if limit
+        q[:exclusive_start_key] = exclusive_start_key if exclusive_start_key
         q[:table_name]     = table_name
         q[:key_conditions] = key_conditions
         q[:query_filter]   = query_filter
@@ -557,7 +620,7 @@ module Dynamoid
       # @since 1.0.0
       #
       # @todo: Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#scan-instance_method
-      def scan(table_name, scan_hash, select_opts = {})
+      def scan(table_name, scan_hash = {}, select_opts = {})
         request = { table_name: table_name }
         request[:consistent_read] = true if select_opts.delete(:consistent_read)
 
@@ -565,15 +628,16 @@ module Dynamoid
         record_limit = select_opts.delete(:record_limit)
         scan_limit = select_opts.delete(:scan_limit)
         batch_size = select_opts.delete(:batch_size)
+        exclusive_start_key = select_opts.delete(:exclusive_start_key)
         request_limit = [record_limit, scan_limit, batch_size].compact.min
         request[:limit] = request_limit if request_limit
-
+        request[:exclusive_start_key] = exclusive_start_key if exclusive_start_key
+        
         if scan_hash.present?
           request[:scan_filter] = scan_hash.reduce({}) do |memo, (attr, cond)|
-            # Flatten as BETWEEN operator specifies array of two elements
             memo.merge(attr.to_s => {
               comparison_operator: FIELD_MAP[cond.keys[0]],
-              attribute_value_list: [cond.values[0].freeze].flatten
+              attribute_value_list: attribute_value_list(FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
             })
           end
         end
@@ -661,7 +725,7 @@ module Dynamoid
           check = {again: true}
           while check[:again]
             sleep Dynamoid::Config.sync_retry_wait_seconds
-            resp = client.describe_table({ table_name: table_name })
+            resp = client.describe_table(table_name: table_name)
             check = check_table_status?(counter, resp, status)
             Dynamoid.logger.info "Checked table status for #{table_name} (check #{check.inspect})"
             counter += 1
@@ -689,7 +753,7 @@ module Dynamoid
         end
       end
 
-      #Converts from symbol to the API string for the given data type
+      # Converts from symbol to the API string for the given data type
       # E.g. :number -> 'N'
       def api_type(type)
         case(type)
@@ -714,13 +778,17 @@ module Dynamoid
       # @return an Expected stanza for the given conditions hash
       #
       def expected_stanza(conditions = nil)
-        expected = Hash.new { |h,k| h[k] = {} }
+        expected = Hash.new { |h, k| h[k] = {} }
         return expected unless conditions
 
         conditions.delete(:unless_exists).try(:each) do |col|
           expected[col.to_s][:exists] = false
         end
-        conditions.delete(:if).try(:each) do |col,val|
+        conditions.delete(:if_exists).try(:each) do |col, val|
+          expected[col.to_s][:exists] = true
+          expected[col.to_s][:value] = val
+        end
+        conditions.delete(:if).try(:each) do |col, val|
           expected[col.to_s][:value] = val
         end
 
@@ -741,7 +809,7 @@ module Dynamoid
       #
       def result_item_to_hash(item)
         {}.tap do |r|
-          item.each { |k,v| r[k.to_sym] = v }
+          item.each { |k, v| r[k.to_sym] = v }
         end
       end
 
@@ -770,10 +838,10 @@ module Dynamoid
         key_schema = aws_key_schema(index.hash_key_schema, index.range_key_schema)
 
         hash = {
-          :index_name => index.name,
-          :key_schema => key_schema,
-          :projection => {
-            :projection_type => index.projection_type.to_s.upcase
+          index_name: index.name,
+          key_schema: key_schema,
+          projection: {
+            projection_type: index.projection_type.to_s.upcase
           }
         }
 
@@ -785,8 +853,8 @@ module Dynamoid
         # Only global secondary indexes have a separate throughput.
         if index.type == :global_secondary
           hash[:provisioned_throughput] = {
-            :read_capacity_units => index.read_capacity,
-            :write_capacity_units => index.write_capacity
+            read_capacity_units: index.read_capacity,
+            write_capacity_units: index.write_capacity
           }
         end
         hash
@@ -856,7 +924,6 @@ module Dynamoid
         attribute_definitions
       end
 
-
       # Builds an attribute definitions based on hash key and range key
       # @params [Hash] hash_key_schema - eg: {:id => :string}
       # @params [Hash] range_key_schema - eg: {:created_at => :datetime}
@@ -887,11 +954,26 @@ module Dynamoid
         aws_type = api_type(dynamoid_type)
 
         {
-          :attribute_name => name.to_s,
-          :attribute_type => aws_type
+          attribute_name: name.to_s,
+          attribute_type: aws_type
         }
       end
 
+      # Build an array of values for Condition
+      # Is used in ScanFilter and QueryFilter
+      # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
+      # @params [String] operator: value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
+      # @params [Object] value: scalar value or array/set
+      def attribute_value_list(operator, value)
+        # For BETWEEN and IN operators we should keep value as is (it should be already an array)
+        # For all the other operators we wrap the value with array
+        if ["BETWEEN", "IN"].include?(operator)
+          [value].flatten
+        else
+          [value]
+        end
+      end
+
       #
       # Represents a table. Exposes data from the "DescribeTable" API call, and also
       # provides methods for coercing values to the proper types based on the table's schema data
@@ -913,7 +995,7 @@ module Dynamoid
         def range_type
           range_type ||= schema[:attribute_definitions].find { |d|
             d[:attribute_name] == range_key
-          }.try(:fetch,:attribute_type, nil)
+          }.try(:fetch, :attribute_type, nil)
         end
 
         def hash_key
@@ -982,19 +1064,19 @@ module Dynamoid
         def to_h
           ret = {}
 
-          @additions.each do |k,v|
+          @additions.each do |k, v|
             ret[k.to_s] = {
               action: ADD,
               value: v
             }
           end
-          @deletions.each do |k,v|
+          @deletions.each do |k, v|
             ret[k.to_s] = {
               action: DELETE,
               value: v
             }
           end
-          @updates.each do |k,v|
+          @updates.each do |k, v|
             ret[k.to_s] = {
               action: PUT,
               value: v
@@ -1004,9 +1086,15 @@ module Dynamoid
           ret
         end
 
-        ADD    = "ADD".freeze
-        DELETE = "DELETE".freeze
-        PUT    = "PUT".freeze
+        ADD    = 'ADD'.freeze
+        DELETE = 'DELETE'.freeze
+        PUT    = 'PUT'.freeze
+      end
+
+      def sanitize_item(attributes)
+        attributes.reject do |k, v|
+          v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
+        end
       end
     end
   end