dynamoid 1.3.3 → 1.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: decc1a7abee20de61693c51cdc89d39dc5a37340
-  data.tar.gz: 33dc108ac41878f619092ca53b3e0865b73aa775
+  metadata.gz: 563c512230d5a7f2485b6cf226eff77accfa2a3e
+  data.tar.gz: e2f88e774032ddf472bc81a2b9e2a5844b61933a
 SHA512:
-  metadata.gz: 21b9a070f8cb22b2c0fb21803e57fca01647c4b0091b98a08f00e84939fe20ed565e3703f78dbd726a62b203da8a36dfa2ded2274186aae43846513b5a64f092
-  data.tar.gz: acf6e06bc0df07455152b26af7830d57cf3b9a13cb670fbe68d3a88de3097cd47fd86e10864dfc6101617d8e0afb65e05e4487c1b3ab1aff6c349a4c4e2d3fcf
+  metadata.gz: 4a9dba2ea759a74d83f27ef984e3ca963230ddebde577084e89324eb69dd08c65c4c19690891e4086bc9cacadb17eaed5c460c03944dcc9db1eb109044ceef91
+  data.tar.gz: 314ffb14350619c950bd6615d46ccb3ca32383116f1a1dfe0aa39891694575e89d3376051bfac53b4e4a0f584257c0315a334365e5715727bcc767a31c2daa5e

data/.gitignore

@@ -66,3 +66,6 @@ Gemfile.lock
 /tmp/
 /spec/DynamoDBLocal-latest/
 /vendor/
+
+# For vagrant
+.vagrant

data/.travis.yml

@@ -2,10 +2,10 @@ language: ruby
 rvm:
   - ruby-2.0.0-p648
   - ruby-2.1.10
-  - ruby-2.2.6
-  - ruby-2.3.3
+  - ruby-2.2.7
+  - ruby-2.3.4
   - ruby-2.4.1
-  - jruby-9.1.8.0
+  - jruby-9.1.9.0
 gemfile:
   - gemfiles/rails_4_0.gemfile
   - gemfiles/rails_4_1.gemfile
@@ -21,7 +21,7 @@ matrix:
       gemfile: gemfiles/rails_4_0.gemfile
     - rvm: ruby-2.4.1
       gemfile: gemfiles/rails_4_1.gemfile
-before_install: gem install bundler -v 1.14.6
+before_install: gem install bundler -v 1.15.4
 install:
   - wget http://dynamodb-local.s3-website-us-west-2.amazonaws.com/dynamodb_local_latest.zip --quiet -O spec/dynamodb_temp.zip
   - unzip -qq spec/dynamodb_temp.zip -d spec/DynamoDBLocal-latest

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 # HEAD
 
+# 1.3.4
+
+Improving
+
+* Added `Chain#last` method (@andrykonchin)
+* Added `date` field type (@andrykonchin)
+* Added `application_timezone` config option (@andrykonchin)
+* Allow consistent reading for Scan request (@andrykonchin)
+* Use Query instead of Scan if there are no conditions for sort (range) key in where clause (@andrykonchin)
+* Support condition operators for non-key fields for Query request (@andrykonchin)
+* Support condition operators for Scan request (@andrykonchin)
+* Support additional operators `in`, `contains`, `not_contains` (@andrykonchin)
+* Support type casting in `where` clause (@andrykonchin)
+* Rename `Chain#eval_limit` to `#record_limit` (@richardhsu)
+* Add `Chain#scan_limit` (@richardhsu)
+* Support batch loading for Query requests (@richardhsu)
+* Support querying Global/Local Secondary Indices in `where` clause (@richardhsu)
+* Only query on GSI if projects all attributes in `where` clause (@richardhsu)
+
+Fixes
+
+* Fix incorrect applying of default field value (#36 and #117, @andrykonchin)
+* Fix sync table creation/deletion (#160, @mirokuxy)
+* Allow to override document timestamps (@andrykonchin)
+* Fix storing empty array as nil (#8, @andrykonchin)
+* Fix `limit` handling for Query requests (#85, @richardhsu)
+* Fix `limit` handling for Scan requests (#85, @richardhsu)
+* Fix paginating for Query requests (@richardhsu)
+* Fix paginating for Scan requests (@richardhsu)
+* Fix `batch_get_item` method call for integer partition key (@mudasirraza)
+
 # 1.3.3
 
 * Allow configuration of the Dynamoid models directory, as not everyone keeps non AR models in app/models

data/README.md

@@ -70,11 +70,7 @@ Then you need to initialize Dynamoid config to get it going. Put code similar to
 
 ```ruby
   Dynamoid.configure do |config|
-    config.adapter = 'aws_sdk_v2' # This adapter establishes a connection to the DynamoDB servers using Amazon's own AWS gem.
     config.namespace = "dynamoid_app_development" # To namespace tables created by Dynamoid from other tables you might have. Set to nil to avoid namespacing.
-    config.warn_on_scan = true # Output a warning to the logger when you perform a scan rather than a query on a table.
-    config.read_capacity = 5 # Read capacity for your tables
-    config.write_capacity = 5 # Write capacity for your tables
     config.endpoint = 'http://localhost:3000' # [Optional]. If provided, it communicates with the DB listening at the endpoint. This is useful for testing with [Amazon Local DB] (http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html).
   end
 
@@ -123,7 +119,8 @@ These fields will not change an existing table: so specifying a new read_capacit
 You'll have to define all the fields on the model and the data type of each field. Every field on the object must be included here; if you miss any they'll be completely bypassed during DynamoDB's initialization and will not appear on the model objects.
 
 By default, fields are assumed to be of type ```:string```. Other built-in types are
-```:integer```, ```:number```, ```:set```, ```:array```, ```:datetime```, ```:boolean```, and ```:serialized```.
+```:integer```, ```:number```, ```:set```, ```:array```, ```:datetime```, ```date```, ```:boolean```, ```:raw``` and ```:serialized```.
+```raw``` type means you can store Ruby Array, Hash, String and numbers.
 If built-in types do not suit you, you can use a custom field type represented by an arbitrary class, provided that the class supports a compatible serialization interface.
 The primary use case for using a custom field type is to represent your business logic with high-level types, while ensuring portability or backward-compatibility of the serialized representation.
 
@@ -257,6 +254,12 @@ end
 
 To see more usage and examples of ActiveModel validations, check out the [ActiveModel validation documentation](http://api.rubyonrails.org/classes/ActiveModel/Validations.html).
 
+If you want to bypass model validation, pass `validate: false` to `save` call:
+
+```ruby
+model.save(validate: false)
+```
+
 ### Callbacks
 
 Dynamoid also employs ActiveModel callbacks. Right now, callbacks are defined on ```save```, ```update```, ```destroy```, which allows you to do ```before_``` or ```after_``` any of those.
@@ -273,6 +276,29 @@ class User
 end
 ```
 
+### STI
+
+Dynamoid supports STI (Single Table Inheritance) like Active Record does. You need just specify `type` field in a base class. Example:
+
+```ruby
+class Animal
+  include Dynamoid::Document
+
+  field :name
+  field :type
+end
+
+class Cat < Animal
+  field :lives, :integer
+end
+
+cat = Cat.create(name: 'Morgan')
+animal = Animal.find(cat.id)
+animal.class
+#=>  Cat
+
+```
+
 ## Usage
 
 ### Object Creation
@@ -317,18 +343,64 @@ u.addresses.where(:city => 'Chicago').all
 
 But keep in mind Dynamoid -- and document-based storage systems in general -- are not drop-in replacements for existing relational databases. The above query does not efficiently perform a conditional join, but instead finds all the user's addresses and naively filters them in Ruby. For large associations this is a performance hit compared to relational database engines.
 
-You can also limit the number of evaluated records, or select a record from which to start, to support pagination:
+#### Limits
+
+There are three types of limits that you can query with:
+
+1. `record_limit` - The number of evaluated records that are returned by the query.
+2. `scan_limit` - The number of scanned records that DynamoDB will look at before returning.
+3. `batch_size` - The number of records requested to DynamoDB per underlying request, good for large queries!
+
+Using these in various combinations results in the underlying requests to be made in the smallest size possible and
+the query returns once `record_limit` or `scan_limit` is satisfied. It will attempt to batch whenever possible.
+
+You can thus limit the number of evaluated records, or select a record from which to start, to support pagination:
 
 ```ruby
-Address.eval_limit(5).start(address) # Only 5 addresses.
+Address.record_limit(5).start(address) # Only 5 addresses starting at `address`
+```
+
+If you are potentially running over a large data set and this is especially true when using certain filters, you may
+want to consider limiting the number of scanned records (the number of records DynamoDB infrastructure looks through
+when evaluating data to return):
+
+```ruby
+Address.scan_limit(5).start(address) # Only scan at most 5 records and return what's found starting from `address`
 ```
 
 For large queries that return many rows, Dynamoid can use AWS' support for requesting documents in batches:
 
 ```ruby
-#Do some maintenance on the entire table without flooding DynamoDB
+# Do some maintenance on the entire table without flooding DynamoDB
 Address.all(batch_size: 100).each { |address| address.do_some_work; sleep(0.01) }
-Address.limit(10_000).batch(100). each { … } #batch specified as part of a chain
+Address.record_limit(10_000).batch(100). each { … } # Batch specified as part of a chain
+```
+
+The implication of batches is that the underlying requests are done in the batch sizes to make the request and responses
+more manageable. Note that this batching is for `Query` and `Scans` and not `BatchGetItem` commands.
+
+#### Sort Conditions and Filters
+
+You are able to optimize query with condition for sort key. Following operators are available: `gt`, `lt`, `gte`, `lte`,
+`begins_with`, `between` as well as equality:
+
+```ruby
+Address.where(latitude: 10212)
+Address.where('latitude.gt': 10212)
+Address.where('latitude.lt': 10212)
+Address.where('latitude.gte': 10212)
+Address.where('latitude.lte': 10212)
+Address.where('city.begins_with': 'Lon')
+Address.where('latitude.between': [10212, 20000])
+```
+
+You are able to filter results on the DynamoDB side and specify conditions for non-key fields.
+Following operators are available: `in`, `contains`, `not_contains`:
+
+```ruby
+Address.where('city.in': ['London', 'Edenburg', 'Birmingham'])
+Address.where('city.contains': [on])
+Address.where('city.not_contains': [ing])
 ```
 
 ### Consistent Reads
@@ -353,7 +425,12 @@ It also supports .gte and .lte. Turning those into symbols and allowing a Rails
 
 ### Global Secondary Indexes
 
-The query I use is as follows, but I really do not know a lot about Dynamoid, and got this working by reading through other Amazon Dynamo code bases and the documentation form Amazon.
+There are two ways to query Global Secondary Indexes (GSI).
+
+#### Explicit
+
+The first way explicitly uses your GSI and utilizes the `find_all_by_secondary_index` method which will lookup a valid
+GSI to use based on the inputs, you MUST provide the correct keys to match the GSI you want:
 
 ```ruby
 find_all_by_secondary_index(
@@ -368,7 +445,8 @@ find_all_by_secondary_index(
     :scan_index_forward => false # or true
 )
 ```
-where the range modifier is one of Dynamoid::Finders::RANGE_MAP.keys, where the RANGE_MAP is:
+
+Where the range modifier is one of `Dynamoid::Finders::RANGE_MAP.keys`, where the `RANGE_MAP` is:
 
 ```ruby
 RANGE_MAP = {
@@ -384,6 +462,53 @@ RANGE_MAP = {
 
 Most range searches, like `eq`, need a single value, and searches like `between`, need an array with two values.
 
+#### Implicit
+
+The second way implicitly uses your GSI through the `where` clauses and deduces the index based on the query fields
+provided. Another added benefit is that it is built into query chaining so you can use all the methods used in normal
+querying. The explicit way from above would be rewritten as follows:
+
+```ruby
+where(dynamo_primary_key_column_name => dynamo_primary_key_value,
+      "#{range_column}.#{range_modifier}" => range_value)
+  .scan_index_forward(false)
+```
+
+The only caveat with this method is that because it is also used for general querying, it WILL NOT use a GSI unless it
+explicitly has defined `projected_attributes: :all` on the GSI in your model. This is because GSIs that do not have all
+attributes projected will only contain the index keys and therefore will not return objects with fully resolved field
+values. It currently opts to provide the complete results rather than partial results unless you've explicitly looked up
+the data.
+
+*Future TODO could involve implementing `select` in chaining as well as resolving the fields with a second query against
+the table since a query against GSI then a query on base table is still likely faster than scan on the base table*
+
+## Configuration
+
+There are listed all the configuration options:
+
+* `adapter` - usefull only for the gem developers to switch to a new adapter. Default and the only available value is `aws_sdk_v2`
+* `namespace` - prefix for table names, default is `dynamoid_#{application_name}_#{environment}` for Rails application and `dynamoid` otherwise
+* `logger` - by default it's a `Rails.logger` in Rails application and `stdout` otherwise. You can disable logging by setting `nil` or `false` values. Set `true` value to use defaults
+* `access_key` - DynamoDb custom credentials for AWS, override global AWS credentials if they present
+* `secret_key` - DynamoDb custom credentials for AWS, override global AWS credentials if they present
+* `region` - DynamoDb custom credentials for AWS, override global AWS credentials if they present
+* `batch_size` - when you try to load multiple items at once with `batch_get_item` call Dynamoid loads them not with one api call but piece by piece. Default is 100 items
+* `read_capacity` - is used at table or indices creation. Default is 100 (units)
+* `write_capacity` - is used at table or indices creation. Default is 20 (units)
+* `warn_on_scan` - log warnings when scan table. Default is `true`
+* `endpoint` - if provided, it communicates with the DynamoDB listening at the endpoint. This is useful for testing with [Amazon Local DB]
+* `identity_map` - ensures that each object gets loaded only once by keeping every loaded object in a map. Looks up objects using the map when referring to them. Isn't thread safe. Default is `false`.
+  `Use Dynamoid::Middleware::IdentityMap` to clear identity map for each HTTP request
+* `timestamps` - by default Dynamoid sets `created_at` and `updated_at` fields for model at creation and updating. You can disable this behavior by setting `false` value
+* `sync_retry_max_times` - when Dynamoid creates or deletes table synchronously it checks for completion specified times. Default is 60 (times). It's a bit over 2 minutes by default
+* `sync_retry_wait_seconds` - time to wait between retries. Default is 2 (seconds)
+* `convert_big_decimal` - if `true` then Dynamoid converts numbers stored in `Hash` in `raw` field to float. Default is `false`
+* `models_dir` - `dynamoid:create_tables` rake task loads DynamoDb models from this directory. Default is `app/models`. In Rails application you should set `./app/models` value
+* `application_timezone` - Dynamoid converts all `datetime` fields to specified time zone when loads data from the storage.
+  Acceptable values - `utc`, `local` (to use system time zone) and time zone name e.g. `Eastern Time (US & Canada)`. Default is `local`
+
+
 ## Concurrency
 
 Dynamoid supports basic, ActiveRecord-like optimistic locking on save operations. Simply add a `lock_version` column to your table like so:

data/Vagrantfile

@@ -0,0 +1,27 @@
+Vagrant.configure('2') do |config|
+  # Choose base box
+  config.vm.box = 'bento/ubuntu-16.04'
+
+  config.vm.provider 'virtualbox' do |vb|
+    # Prevent clock skew when host goes to sleep while VM is running
+    vb.customize ['guestproperty', 'set', :id, '/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold', 10_000]
+
+    vb.cpus = 2
+    vb.memory = 2048
+  end
+
+  # Defaults
+  config.vm.provision :salt do |salt|
+    salt.masterless = true
+    salt.minion_config = '.dev/vagrant/minion'
+
+    # Pillars
+    salt.pillar({
+      'ruby' => {
+        'version' => '2.3.3',
+      }
+    })
+
+    salt.run_highstate = true
+  end
+end

data/dynamoid.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
       "LICENSE.txt",
       "README.md"
   ]
-  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(bin|test|spec|features)/}) }
+  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(bin|test|spec|features|.dev|Vagrantfile)/}) }
   spec.homepage = "http://github.com/Dynamoid/Dynamoid"
   spec.licenses = ["MIT"]
   spec.bindir = "exe"

data/lib/dynamoid.rb

@@ -36,8 +36,6 @@ end
 module Dynamoid
   extend self
 
-  MAX_ITEM_SIZE = 65_536
-
   def configure
     block_given? ? yield(Dynamoid::Config) : Dynamoid::Config
   end

data/lib/dynamoid/adapter.rb

@@ -145,7 +145,7 @@ module Dynamoid
       #
       # @since 0.2.0
       define_method(m) do |*args|
-        benchmark("#{m.to_s}", args) {adapter.send(m, *args)}
+        benchmark("#{m.to_s}", *args) {adapter.send(m, *args)}
       end
     end
 

data/lib/dynamoid/adapter_plugin/aws_sdk_v2.rb

@@ -13,6 +13,21 @@ module Dynamoid
           range_between:      'BETWEEN',
           range_eq:           'EQ'
       }
+
+      # Don't implement NULL and NOT_NULL because it doesn't make seanse -
+      # we declare schema in models
+      FIELD_MAP = {
+          eq:           'EQ',
+          gt:           'GT',
+          lt:           'LT',
+          gte:          'GE',
+          lte:          'LE',
+          begins_with:  'BEGINS_WITH',
+          between:      'BETWEEN',
+          in:           'IN',
+          contains:     'CONTAINS',
+          not_contains: 'NOT_CONTAINS'
+      }
       HASH_KEY  = "HASH".freeze
       RANGE_KEY = "RANGE".freeze
       STRING_TYPE  = "S".freeze
@@ -110,12 +125,12 @@ module Dynamoid
       # @todo: Provide support for passing options to underlying batch_get_item http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
       def batch_get_item(table_ids, options = {})
         request_items = Hash.new{|h, k| h[k] = []}
-        return request_items if table_ids.all?{|k, v| v.empty?}
+        return request_items if table_ids.all?{|k, v| v.blank?}
 
         ret = Hash.new([].freeze) # Default for tables where no rows are returned
 
         table_ids.each do |t, ids|
-          next if ids.empty?
+          next if ids.blank?
           tbl = describe_table(t)
           hk  = tbl.hash_key.to_s
           rng = tbl.range_key.to_s
@@ -230,9 +245,9 @@ module Dynamoid
         end
         resp = client.create_table(client_opts)
         options[:sync] = true if !options.has_key?(:sync) && ls_indexes.present? || gs_indexes.present?
-        until_past_table_status(table_name) if options[:sync] &&
+        until_past_table_status(table_name, :creating) if options[:sync] &&
             (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
-            status != TABLE_STATUSES[:creating]
+            status == TABLE_STATUSES[:creating]
         # Response to original create_table, which, if options[:sync]
         #   may have an outdated table_description.table_status of "CREATING"
         resp
@@ -295,7 +310,7 @@ module Dynamoid
         resp = client.delete_table(table_name: table_name)
         until_past_table_status(table_name, :deleting) if options[:sync] &&
             (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
-            status != TABLE_STATUSES[:deleting]
+            status == TABLE_STATUSES[:deleting]
         table_cache.delete(table_name)
       rescue Aws::DynamoDB::Errors::ResourceInUseException => e
         Dynamoid.logger.error "Table #{table_name} cannot be deleted as it is in use"
@@ -382,7 +397,7 @@ module Dynamoid
         options ||= {}
 
         object.each do |k, v|
-          next if v.nil? || (v.respond_to?(:empty?) && v.empty?)
+          next if v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
           item[k.to_s] = v
         end
 
@@ -419,28 +434,37 @@ module Dynamoid
       # @todo Provide support for various other options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#query-instance_method
       def query(table_name, opts = {})
         table = describe_table(table_name)
-        hk    = (opts[:hash_key].present? ? opts[:hash_key] : table.hash_key).to_s
-        rng   = (opts[:range_key].present? ? opts[:range_key] : table.range_key).to_s
+        hk    = (opts[:hash_key].present? ? opts.delete(:hash_key) : table.hash_key).to_s
+        rng   = (opts[:range_key].present? ? opts.delete(:range_key) : table.range_key).to_s
         q     = opts.slice(
                   :consistent_read,
                   :scan_index_forward,
-                  :limit,
                   :select,
                   :index_name
                 )
 
         opts.delete(:consistent_read)
         opts.delete(:scan_index_forward)
-        opts.delete(:limit)
         opts.delete(:select)
         opts.delete(:index_name)
 
+        # Deal with various limits and batching
+        record_limit = opts.delete(:record_limit)
+        scan_limit = opts.delete(:scan_limit)
+        batch_size = opts.delete(:batch_size)
+        limit = [record_limit, scan_limit, batch_size].compact.min
+        q[:limit] = limit if limit
+
         opts.delete(:next_token).tap do |token|
           break unless token
           q[:exclusive_start_key] = {
             hk  => token[:hash_key_element],
             rng => token[:range_key_element]
           }
+          # For secondary indices the start key must contain the indices composite key
+          # but also the table's composite keys
+          q[:exclusive_start_key][table.hash_key] = token[:table_hash_key_element] if token[:table_hash_key_element]
+          q[:exclusive_start_key][table.range_key] = token[:table_range_key_element] if token[:table_range_key_element]
         end
 
         key_conditions = {
@@ -464,14 +488,55 @@ module Dynamoid
           }
         end
 
+        query_filter = {}
+        opts.reject {|k,_| k.in? RANGE_MAP.keys}.each do |attr, hash|
+          query_filter[attr] = {
+            comparison_operator: FIELD_MAP[hash.keys[0]],
+            attribute_value_list: [
+              hash.values[0].freeze
+            ].flatten # Flatten as BETWEEN operator specifies array of two elements
+          }
+        end
+
         q[:table_name]     = table_name
         q[:key_conditions] = key_conditions
+        q[:query_filter]   = query_filter
 
         Enumerator.new { |y|
+          record_count = 0
+          scan_count = 0
           loop do
+            # Adjust the limit down if the remaining record and/or scan limit are
+            # lower to obey limits. We can assume the difference won't be
+            # negative due to break statements below but choose smaller limit
+            # which is why we have 2 separate if statements.
+            # NOTE: Adjusting based on record_limit can cause many HTTP requests
+            # being made. We may want to change this behavior, but it affects
+            # filtering on data with potentially large gaps.
+            # Example:
+            #    User.where('created_at.gte' => 1.day.ago).record_limit(1000)
+            #    Records 1-999 User's that fit criteria
+            #    Records 1000-2000 Users's that do not fit criteria
+            #    Record 2001 fits criteria
+            # The underlying implementation will have 1 page for records 1-999
+            # then will request with limit 1 for records 1000-2000 (making 1000
+            # requests of limit 1) until hit record 2001.
+            if q[:limit] && record_limit && record_limit - record_count < q[:limit]
+              q[:limit] = record_limit - record_count
+            end
+            if q[:limit] && scan_limit && scan_limit - scan_count < q[:limit]
+              q[:limit] = scan_limit - scan_count
+            end
+
             results = client.query(q)
             results.items.each { |row| y << result_item_to_hash(row) }
 
+            record_count += results.items.size
+            break if record_limit && record_count >= record_limit
+
+            scan_count += results.scanned_count
+            break if scan_limit && scan_count >= scan_limit
+
             if(lk = results.last_evaluated_key)
               q[:exclusive_start_key] = lk
             else
@@ -493,28 +558,63 @@ module Dynamoid
       #
       # @todo: Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#scan-instance_method
       def scan(table_name, scan_hash, select_opts = {})
-        limit = select_opts.delete(:limit)
-        batch = select_opts.delete(:batch_size)
-
         request = { table_name: table_name }
-        request[:limit] = batch || limit if batch || limit
-        request[:scan_filter] = scan_hash.reduce({}) do |memo, kvp|
-          memo[kvp[0].to_s] = {
-            attribute_value_list: [kvp[1]],
-            # TODO: Provide support for all comparison operators
-            comparison_operator: EQ
-          }
-          memo
-        end if scan_hash.present?
+        request[:consistent_read] = true if select_opts.delete(:consistent_read)
+
+        # Deal with various limits and batching
+        record_limit = select_opts.delete(:record_limit)
+        scan_limit = select_opts.delete(:scan_limit)
+        batch_size = select_opts.delete(:batch_size)
+        request_limit = [record_limit, scan_limit, batch_size].compact.min
+        request[:limit] = request_limit if request_limit
+
+        if scan_hash.present?
+          request[:scan_filter] = scan_hash.reduce({}) do |memo, (attr, cond)|
+            # Flatten as BETWEEN operator specifies array of two elements
+            memo.merge(attr.to_s => {
+              comparison_operator: FIELD_MAP[cond.keys[0]],
+              attribute_value_list: [cond.values[0].freeze].flatten
+            })
+          end
+        end
 
         Enumerator.new do |y|
-          # Batch loop, pulls multiple requests until done using the start_key
+          record_count = 0
+          scan_count = 0
           loop do
+            # Adjust the limit down if the remaining record and/or scan limit are
+            # lower to obey limits. We can assume the difference won't be
+            # negative due to break statements below but choose smaller limit
+            # which is why we have 2 separate if statements.
+            # NOTE: Adjusting based on record_limit can cause many HTTP requests
+            # being made. We may want to change this behavior, but it affects
+            # filtering on data with potentially large gaps.
+            # Example:
+            #    User.where('created_at.gte' => 1.day.ago).record_limit(1000)
+            #    Records 1-999 User's that fit criteria
+            #    Records 1000-2000 Users's that do not fit criteria
+            #    Record 2001 fits criteria
+            # The underlying implementation will have 1 page for records 1-999
+            # then will request with limit 1 for records 1000-2000 (making 1000
+            # requests of limit 1) until hit record 2001.
+            if request[:limit] && record_limit && record_limit - record_count < request[:limit]
+              request[:limit] = record_limit - record_count
+            end
+            if request[:limit] && scan_limit && scan_limit - scan_count < request[:limit]
+              request[:limit] = scan_limit - scan_count
+            end
+
             results = client.scan(request)
+            results.items.each { |row| y << result_item_to_hash(row) }
+
+            record_count += results.items.size
+            break if record_limit && record_count >= record_limit
 
-            results.data[:items].each { |row| y << result_item_to_hash(row) }
+            scan_count += results.scanned_count
+            break if scan_limit && scan_count >= scan_limit
 
-            if((lk = results[:last_evaluated_key]) && batch)
+            # Keep pulling if we haven't finished paging in all data
+            if(lk = results[:last_evaluated_key])
               request[:exclusive_start_key] = lk
             else
               break

data/lib/dynamoid/associations.rb

@@ -19,7 +19,7 @@ module Dynamoid
     
     # Create the association tracking attribute and initialize it to an empty hash.
     included do
-      class_attribute :associations
+      class_attribute :associations, instance_accessor: false
       
       self.associations = {}
     end

data/lib/dynamoid/config.rb

@@ -13,7 +13,6 @@ module Dynamoid
     # All the default options.
     option :adapter, :default => 'aws_sdk_v2'
     option :namespace, :default => defined?(Rails) ? "dynamoid_#{Rails.application.class.parent_name}_#{Rails.env}" : "dynamoid"
-    option :logger, :default => defined?(Rails)
     option :access_key, :default => nil
     option :secret_key, :default => nil
     option :region, :default => nil
@@ -22,14 +21,13 @@ module Dynamoid
     option :write_capacity, :default => 20
     option :warn_on_scan, :default => true
     option :endpoint, :default => nil
-    option :use_ssl, :default => true
-    option :port, :default => '443'
     option :identity_map, :default => false
     option :timestamps, :default => true
     option :sync_retry_max_times, :default => 60 # a bit over 2 minutes
     option :sync_retry_wait_seconds, :default => 2
     option :convert_big_decimal, :default => false
     option :models_dir, :default => "app/models" # perhaps you keep your dynamoid models in a different directory?
+    option :application_timezone, default: :local # available values - :utc, :local, time zone names
 
     # The default logger for Dynamoid: either the Rails logger or just stdout.
     #

data/lib/dynamoid/criteria.rb

@@ -6,11 +6,11 @@ module Dynamoid
   # Allows classes to be queried by where, all, first, and each and return criteria chains.
   module Criteria
     extend ActiveSupport::Concern
-    
+
     module ClassMethods
-      
-      [:where, :all, :first, :each, :eval_limit, :start, :scan_index_forward].each do |meth|
-        # Return a criteria chain in response to a method that will begin or end a chain. For more information, 
+
+      [:where, :all, :first, :last, :each, :record_limit, :scan_limit, :batch, :start, :scan_index_forward].each do |meth|
+        # Return a criteria chain in response to a method that will begin or end a chain. For more information,
         # see Dynamoid::Criteria::Chain.
         #
         # @since 0.2.0
@@ -25,5 +25,5 @@ module Dynamoid
       end
     end
   end
-  
+
 end

data/lib/dynamoid/criteria/chain.rb

@@ -6,8 +6,9 @@ module Dynamoid #:nodoc:
     # chain to relation). It is a chainable object that builds up a query and eventually executes it by a Query or Scan.
     class Chain
       # TODO: Should we transform any other types of query values?
-      TYPES_TO_DUMP_FOR_QUERY = [:string, :integer, :boolean]
+      TYPES_TO_DUMP_FOR_QUERY = [:string, :integer, :boolean, :serialized]
       attr_accessor :query, :source, :values, :consistent_read
+      attr_reader :hash_key, :range_key, :index_name
       include Enumerable
       # Create a new criteria chain.
       #
@@ -54,6 +55,12 @@ module Dynamoid #:nodoc:
         records
       end
 
+      # Returns the last fetched record matched the criteria
+      #
+      def last
+        all.last
+      end
+
       # Destroys all the records matching the criteria.
       #
       def destroy_all
@@ -76,8 +83,19 @@ module Dynamoid #:nodoc:
         end
       end
 
-      def eval_limit(limit)
-        @eval_limit = limit
+      # The record limit is the limit of evaluated records returned by the
+      # query or scan.
+      def record_limit(limit)
+        @record_limit = limit
+        self
+      end
+
+      # The scan limit which is the limit of records that DynamoDB will
+      # internally query or scan. This is different from the record limit
+      # as with filtering DynamoDB may look at N scanned records but return 0
+      # records if none pass the filter.
+      def scan_limit(limit)
+        @scan_limit = limit
         self
       end
 
@@ -139,26 +157,21 @@ module Dynamoid #:nodoc:
       def records_via_scan
         if Dynamoid::Config.warn_on_scan
           Dynamoid.logger.warn 'Queries without an index are forced to use scan and are generally much slower than indexed queries!'
-          Dynamoid.logger.warn "You can index this query by adding this to #{source.to_s.downcase}.rb: index [#{source.attributes.sort.collect{|attr| ":#{attr}"}.join(', ')}]"
-        end
-
-        if @consistent_read
-          raise Dynamoid::Errors::InvalidQuery, 'Consistent read is not supported by SCAN operation'
+          Dynamoid.logger.warn "You can index this query by adding this to #{source.to_s.downcase}.rb: index [#{query.keys.sort.collect{|name| ":#{name}"}.join(', ')}]"
         end
 
         Enumerator.new do |yielder|
-          Dynamoid.adapter.scan(source.table_name, query, scan_opts).each do |hash|
+          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).each do |hash|
             yielder.yield source.from_database(hash)
           end
         end
       end
 
       def range_hash(key)
-        val = query[key]
-
-        return { :range_value => query[key] } if query[key].is_a?(Range)
+        name, operation = key.to_s.split('.')
+        val = type_cast_condition_parameter(name, query[key])
 
-        case key.to_s.split('.').last
+        case operation
         when 'gt'
           { :range_greater_than => val }
         when 'lt'
@@ -174,45 +187,170 @@ module Dynamoid #:nodoc:
         end
       end
 
+      def field_hash(key)
+        name, operation = key.to_s.split('.')
+        val = type_cast_condition_parameter(name, query[key])
+
+        hash = case operation
+        when 'gt'
+          { gt: val }
+        when 'lt'
+          { lt: val }
+        when 'gte'
+          { gte: val }
+        when 'lte'
+          { lte: val }
+        when 'between'
+          { between: val }
+        when 'begins_with'
+          { begins_with: val }
+        when 'in'
+          { in: val }
+        when 'contains'
+          { contains: val }
+        when 'not_contains'
+          { not_contains: val }
+        end
+
+        return { name.to_sym => hash }
+      end
+
       def range_query
-        opts = { :hash_value => query[source.hash_key] }
-        query.keys.select { |k| k.to_s.include?('.') }.each do |key|
-          opts.merge!(range_hash(key))
+        opts = {}
+
+        # Add hash key
+        opts[:hash_key] = @hash_key
+        opts[:hash_value] = type_cast_condition_parameter(@hash_key, query[@hash_key])
+
+        # Add range key
+        if @range_key
+          opts[:range_key] = @range_key
+          if query[@range_key].present?
+            value = type_cast_condition_parameter(@range_key, query[@range_key])
+            opts.update(:range_eq => value)
+          end
+
+          query.keys.select { |k| k.to_s =~ /^#{@range_key}\./ }.each do |key|
+            opts.merge!(range_hash(key))
+          end
         end
+
+        (query.keys.map(&:to_sym) - [@hash_key.to_sym, @range_key.try(:to_sym)])
+          .reject { |k, _| k.to_s =~ /^#{@range_key}\./ }
+          .each do |key|
+          if key.to_s.include?('.')
+            opts.update(field_hash(key))
+          else
+            value = type_cast_condition_parameter(key, query[key])
+            opts[key] = {eq: value}
+          end
+        end
+
         opts.merge(query_opts).merge(consistent_opts)
       end
 
-      def query_keys
-        query.keys.collect{|k| k.to_s.split('.').first}
+      def type_cast_condition_parameter(key, value)
+        if !value.respond_to?(:to_ary)
+          source.dump_field(value, source.attributes[key.to_sym])
+        else
+          value.to_ary.map { |el| source.dump_field(el, source.attributes[key.to_sym]) }
+        end
       end
 
-      # [hash_key] or [hash_key, range_key] is specified in query keys.
       def key_present?
-        query_keys == [source.hash_key.to_s] || (query_keys.to_set == [source.hash_key.to_s, source.range_key.to_s].to_set)
+        query_keys = query.keys.collect { |k| k.to_s.split('.').first }
+
+        # See if querying based on table hash key
+        if query_keys.include?(source.hash_key.to_s)
+          @hash_key = source.hash_key
+
+          # Use table's default range key
+          if query_keys.include?(source.range_key.to_s)
+            @range_key = source.range_key
+            return true
+          end
+
+          # See if can use any local secondary index range key
+          # Chooses the first LSI found that can be utilized for the query
+          source.local_secondary_indexes.each do |_, lsi|
+            next unless query_keys.include?(lsi.range_key.to_s)
+            @range_key = lsi.range_key
+            @index_name = lsi.name
+          end
+
+          return true
+        end
+
+        # See if can use any global secondary index
+        # Chooses the first GSI found that can be utilized for the query
+        # But only do so if projects ALL attributes otherwise we won't
+        # get back full data
+        source.global_secondary_indexes.each do |_, gsi|
+          next unless query_keys.include?(gsi.hash_key.to_s) && gsi.projected_attributes == :all
+          @hash_key = gsi.hash_key
+          @range_key = gsi.range_key
+          @index_name = gsi.name
+          return true
+        end
+
+        # Could not utilize any indices so we'll have to scan
+        false
       end
 
+      # Start key needs to be set up based on the index utilized
+      # If using a secondary index then we must include the index's composite key
+      # as well as the tables composite key.
       def start_key
-        key = { :hash_key_element => @start.hash_key }
-        if range_key = @start.class.range_key
-          key.merge!({:range_key_element => @start.send(range_key) })
+        hash_key = @hash_key || source.hash_key
+        range_key = @range_key || source.range_key
+
+        key = {}
+        key[:hash_key_element] = type_cast_condition_parameter(hash_key, @start.send(hash_key))
+        key[:range_key_element] = type_cast_condition_parameter(range_key, @start.send(range_key)) if range_key
+
+        # Add table composite keys if differ from secondary index used composite key
+        if hash_key != source.hash_key
+          key[:table_hash_key_element] = type_cast_condition_parameter(source.hash_key, @start.hash_key)
         end
+        if source.range_key && range_key != source.range_key
+          key[:table_range_key_element] = type_cast_condition_parameter(source.range_key, @start.range_value)
+        end
+
         key
       end
 
       def query_opts
         opts = {}
+        opts[:index_name] = @index_name if @index_name
         opts[:select] = 'ALL_ATTRIBUTES'
-        opts[:limit] = @eval_limit if @eval_limit
+        opts[:record_limit] = @record_limit if @record_limit
+        opts[:scan_limit] = @scan_limit if @scan_limit
+        opts[:batch_size] = @batch_size if @batch_size
         opts[:next_token] = start_key if @start
         opts[:scan_index_forward] = @scan_index_forward
         opts
       end
 
+      def scan_query
+        {}.tap do |opts|
+          query.keys.map(&:to_sym).each do |key|
+            if key.to_s.include?('.')
+              opts.update(field_hash(key))
+            else
+              value = type_cast_condition_parameter(key, query[key])
+              opts[key] = {eq: value}
+            end
+          end
+        end
+      end
+
       def scan_opts
         opts = {}
-        opts[:limit] = @eval_limit if @eval_limit
-        opts[:next_token] = start_key if @start
+        opts[:record_limit] = @record_limit if @record_limit
+        opts[:scan_limit] = @scan_limit if @scan_limit
         opts[:batch_size] = @batch_size if @batch_size
+        opts[:next_token] = start_key if @start
+        opts[:consistent_read] = true if @consistent_read
         opts
       end
     end

data/lib/dynamoid/document.rb

@@ -8,7 +8,7 @@ module Dynamoid #:nodoc:
     include Dynamoid::Components
 
     included do
-      class_attribute :options, :read_only_attributes, :base_class
+      class_attribute :options, :read_only_attributes, :base_class, instance_accessor: false
       self.options = {}
       self.read_only_attributes = []
       self.base_class = self
@@ -120,6 +120,10 @@ module Dynamoid #:nodoc:
     #
     # @since 0.2.0
     def initialize(attrs = {})
+      # we need this hack for Rails 4.0 only
+      # because `run_callbacks` calls `attributes` getter while it is still nil
+      @attributes = {}
+
       run_callbacks :initialize do
         @new_record = true
         @attributes ||= {}

data/lib/dynamoid/fields.rb

@@ -10,12 +10,13 @@ module Dynamoid #:nodoc:
       :number,
       :integer,
       :string,
-      :datetime
+      :datetime,
+      :serialized,
     ]
 
     # Initialize the attributes we know the class has, in addition to our magic attributes: id, created_at, and updated_at.
     included do
-      class_attribute :attributes
+      class_attribute :attributes, instance_accessor: false
       class_attribute :range_key
 
       self.attributes = {}
@@ -30,7 +31,7 @@ module Dynamoid #:nodoc:
       # Specify a field for a document.
       #
       # Its type determines how it is coerced when read in and out of the datastore.
-      # You can specify :integer, :number, :set, :array, :datetime, and :serialized,
+      # You can specify :integer, :number, :set, :array, :datetime, :date and :serialized,
       # or specify a class that defines a serialization strategy.
       #
       # If you specify a class for field type, Dynamoid will serialize using
@@ -51,17 +52,19 @@ module Dynamoid #:nodoc:
         end
         self.attributes = attributes.merge(name => {:type => type}.merge(options))
 
-        define_method(named) { read_attribute(named) }
-        define_method("#{named}?") do
-          value = read_attribute(named)
-          case value
-          when true        then true
-          when false, nil  then false
-          else
-            !value.nil?
+        generated_methods.module_eval do
+          define_method(named) { read_attribute(named) }
+          define_method("#{named}?") do
+            value = read_attribute(named)
+            case value
+            when true        then true
+            when false, nil  then false
+            else
+              !value.nil?
+            end
           end
+          define_method("#{named}=") {|value| write_attribute(named, value) }
         end
-        define_method("#{named}=") {|value| write_attribute(named, value) }
       end
 
       def range(name, type = :string)
@@ -80,9 +83,22 @@ module Dynamoid #:nodoc:
       def remove_field(field)
         field = field.to_sym
         attributes.delete(field) or raise "No such field"
-        remove_method field
-        remove_method :"#{field}="
-        remove_method :"#{field}?"
+
+        generated_methods.module_eval do
+          remove_method field
+          remove_method :"#{field}="
+          remove_method :"#{field}?"
+        end
+      end
+
+      private
+
+      def generated_methods
+        @generated_methods ||= begin
+          Module.new.tap do |mod|
+            include(mod)
+          end
+        end
       end
     end
 
@@ -97,10 +113,6 @@ module Dynamoid #:nodoc:
     #
     # @since 0.2.0
     def write_attribute(name, value)
-      if (size = value.to_s.size) > MAX_ITEM_SIZE
-        Dynamoid.logger.warn "DynamoDB can't store items larger than #{MAX_ITEM_SIZE} and the #{name} field has a length of #{size}."
-      end
-
       if association = @associations[name]
         association.reset
       end
@@ -146,14 +158,16 @@ module Dynamoid #:nodoc:
     #
     # @since 0.2.0
     def set_created_at
-      self.created_at = DateTime.now.in_time_zone(Time.zone) if Dynamoid::Config.timestamps
+      self.created_at ||= DateTime.now.in_time_zone(Time.zone) if Dynamoid::Config.timestamps
     end
 
     # Automatically called during the save callback to set the updated_at time.
     #
     # @since 0.2.0
     def set_updated_at
-      self.updated_at = DateTime.now.in_time_zone(Time.zone) if Dynamoid::Config.timestamps
+      if Dynamoid::Config.timestamps && !self.updated_at_changed?
+        self.updated_at = DateTime.now.in_time_zone(Time.zone)
+      end
     end
 
     def set_type

data/lib/dynamoid/indexes.rb

@@ -3,8 +3,8 @@ module Dynamoid
     extend ActiveSupport::Concern
 
     included do
-      class_attribute :local_secondary_indexes
-      class_attribute :global_secondary_indexes
+      class_attribute :local_secondary_indexes, instance_accessor: false
+      class_attribute :global_secondary_indexes, instance_accessor: false
       self.local_secondary_indexes = {}
       self.global_secondary_indexes = {}
     end

data/lib/dynamoid/persistence.rb

@@ -13,6 +13,8 @@ module Dynamoid
     attr_accessor :new_record
     alias :new_record? :new_record
 
+    UNIX_EPOCH_DATE = Date.new(1970, 1, 1).freeze
+
     module ClassMethods
 
       def table_name
@@ -69,7 +71,15 @@ module Dynamoid
         incoming = (incoming || {}).symbolize_keys
         Hash.new.tap do |hash|
           self.attributes.each do |attribute, options|
-            hash[attribute] = undump_field(incoming[attribute], options)
+            if incoming.has_key?(attribute)
+              hash[attribute] = undump_field(incoming[attribute], options)
+            elsif options.has_key?(:default)
+              default_value = options[:default]
+              value = default_value.respond_to?(:call) ? default_value.call : default_value.dup
+              hash[attribute] = value
+            else
+              hash[attribute] = nil
+            end
           end
           incoming.each {|attribute, value| hash[attribute] = value unless hash.has_key? attribute }
         end
@@ -92,10 +102,6 @@ module Dynamoid
             value
           end
         else
-          if value.nil? && (default_value = options[:default])
-            value = default_value.respond_to?(:call) ? default_value.call : default_value.dup
-          end
-
           unless value.nil?
             case options[:type]
               when :string
@@ -118,7 +124,20 @@ module Dynamoid
                 if value.is_a?(Date) || value.is_a?(DateTime) || value.is_a?(Time)
                   value
                 else
-                  Time.at(value).to_datetime
+                  case Dynamoid::Config.application_timezone
+                    when :utc
+                      ActiveSupport::TimeZone['UTC'].at(value).to_datetime
+                    when :local
+                      Time.at(value).to_datetime
+                    when String
+                      ActiveSupport::TimeZone[Dynamoid::Config.application_timezone].at(value).to_datetime
+                  end
+                end
+              when :date
+                if value.is_a?(Date) || value.is_a?(DateTime) || value.is_a?(Time)
+                  value.to_date
+                else
+                  UNIX_EPOCH_DATE + value.to_i
                 end
               when :boolean
                 # persisted as 't', but because undump is called during initialize it can come in as true
@@ -159,6 +178,8 @@ module Dynamoid
               !value.nil? ? value : nil
             when :datetime
               !value.nil? ? value.to_time.to_f : nil
+            when :date
+              !value.nil? ? (value.to_date - UNIX_EPOCH_DATE).to_i : nil
             when :serialized
               options[:serializer] ? options[:serializer].dump(value) : value.to_yaml
             when :raw
@@ -176,7 +197,7 @@ module Dynamoid
           type.respond_to?(:dynamoid_field_type) ? type.dynamoid_field_type : :string
         else
           case type
-            when :integer, :number, :datetime
+            when :integer, :number, :datetime, :date
               :number
             when :string, :serialized
               :string

data/lib/dynamoid/version.rb

@@ -1,3 +1,3 @@
 module Dynamoid
-  VERSION = "1.3.3"
+  VERSION = "1.3.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamoid
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.4
 platform: ruby
 authors:
 - Josh Symonds
@@ -20,7 +20,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-15 00:00:00.000000000 Z
+date: 2017-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -226,6 +226,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- Vagrantfile
 - dynamoid.gemspec
 - gemfiles/rails_4_0.gemfile
 - gemfiles/rails_4_0.gemfile.lock
@@ -285,7 +286,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: Dynamoid is an ORM for Amazon's DynamoDB