dynamoid 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a6f0df676274e985818f75e1800d007f4f8e296
-  data.tar.gz: 613a8af57da76c4cb7b896e5fa4db86607eb9d26
+  metadata.gz: 66d793bb4519ccd94ce9e8dd46550c02db75390e
+  data.tar.gz: 55f4da7a601691112e2eac068a81f3194b893fa5
 SHA512:
-  metadata.gz: 37716f3cb495c6de57ddc51ef1cbf4933319ba78f003e77a53f69c232268f49709c3a8e420abc9ec7093643328925811e82f68380a821f774e4071b317d37d48
-  data.tar.gz: 2e192af7a28032a4458eaaf293d0bd03704d31094b861859a931e4c81e46ba1ba09e3a3e5b75a6a8befa34f55037d4a90f2f8da389402643e561a18ff79c8a5d
+  metadata.gz: f26bf8ba08816cffae591a78e2b9148abed5423bd8a84c556888bba9c9e35dd24e971b9425e2729ae71be6ec0ee015eab274e321bcfe99fe18243339e13e2e2b
+  data.tar.gz: c2f267ed34a44de9c940416ebbbc950ab6588aa1e75de85d016171d881466613966223c24d59b669fea09919f1b5a47d8fc1005c68692f9b8c4003dd89a0a045

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+# HEAD
+
+# 1.3.0
+
+* Fixed specs (@AlexNisnevich & @pboling)
+* Fix `blank?` and `present?` behavior for single associations (#110, @AlexNisnevich & @bayesimpact)
+* Support BatchGet for more than 100 items (#80, @getninjas)
+* Add ability to specify connection settings specific to Dynamoid (#116, @NielsKSchjoedt)
+* Adds Support for Rails 5! (#109, @gastzars)
+* Table Namespace Fix (#79, @alexperto)
+* Improve Testing Docs (#103, @tadast)
+* Query All Items by Looping (#102, @richardhsu)
+* Store document in DocumentNotValid error for easier debugging (#98, holyketzer)
+* Better support for raw datatype (#104, @OpenGov)
+* Fix associative tables with non-id primary keys (#86, @everett-wetchler)
+
 # 1.2.1
 
 * Remove accidental Gemfile.lock; fix .gitignore (#95, @pboling)

data/README.md

@@ -30,7 +30,7 @@ gem 'aws-sdk', '~>2'
 
 (or) include the aws-sdk in your Gemfile.
 
-**NOTE:** Dynamoid-1.0 doesn't support aws-sdk Version 1 (Use Dynamoid Major Version 0 for aws-sdk 1)   
+**NOTE:** Dynamoid-1.0 doesn't support aws-sdk Version 1 (Use Dynamoid Major Version 0 for aws-sdk 1)
 
 Configure AWS access:
 [Reference](https://github.com/aws/aws-sdk-ruby)
@@ -48,6 +48,16 @@ Create config/initializers/aws.rb as follows:
 
 ```
 
+Alternatively, if you don't want Aws connection settings to be overwritten for you entire project, you can specify connection settings for Dynamoid only, by setting those in the `Dynamoid.configure` clause:
+
+```ruby
+  Dynamoid.configure do |config|
+    config.access_key = 'REPLACE_WITH_ACCESS_KEY_ID'
+    config.secret_key = 'REPLACE_WITH_SECRET_ACCESS_KEY'
+    config.region = 'us-west-2'
+  end
+```
+
 For a full list of the DDB regions, you can go
 [here](http://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region).
 
@@ -60,7 +70,7 @@ Then you need to initialize Dynamoid config to get it going. Put code similar to
     config.warn_on_scan = true # Output a warning to the logger when you perform a scan rather than a query on a table.
     config.read_capacity = 5 # Read capacity for your tables
     config.write_capacity = 5 # Write capacity for your tables
-    config.endpoint = 'http://localhost:3000' # [Optional]. If provided, it communicates with the DB listening at the endpoint. This is useful for testing with [Amazon Local DB] (http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html). 
+    config.endpoint = 'http://localhost:3000' # [Optional]. If provided, it communicates with the DB listening at the endpoint. This is useful for testing with [Amazon Local DB] (http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html).
   end
 
 ```
@@ -129,20 +139,20 @@ To use a custom type for a field, suppose you have a `Money` type.
 ```ruby
   class Money
     # ... your business logic ...
-    
+
     def dynamoid_dump
       "serialized representation as a string"
     end
-    
+
     def self.dynamoid_load(serialized_str)
       # parse serialized representation and return a Money instance
       Money.new(...)
     end
   end
-  
+
   class User
     include Dynamoid::Document
-    
+
     field :balance, Money
   end
 ```
@@ -155,20 +165,20 @@ add a level of indirection for serializing.)  Example:
 ```ruby
   # Third-party Money class
   class Money; end
-  
+
   class MoneyAdapter
     def self.dynamoid_load(money_serialized_str)
       Money.new(...)
     end
-    
+
     def self.dynamoid_dump(money_obj)
       money_obj.value.to_s
     end
   end
-  
+
   class User
     include Dynamoid::Document
-    
+
     field :balance, MoneyAdapter
   end
 ```
@@ -207,12 +217,12 @@ class Address
 
   ...
 
-  belongs_to :address # Automatically links up with the user model
+  belongs_to :user # Automatically links up with the user model
 
 end
 ```
 
-Contrary to what you'd expect, association information is always contained on the object specifying the association, even if it seems like the association has a foreign key. This is a side effect of DynamoDB's structure: it's very difficult to find foreign keys without an index. Usually you won't find this to be a problem, but it does mean that association methods that build new models will not work correctly -- for example, ```user.addresses.new``` returns an address that is not associated to the user. We'll be correcting this soon.
+Contrary to what you'd expect, association information is always contained on the object specifying the association, even if it seems like the association has a foreign key. This is a side effect of DynamoDB's structure: it's very difficult to find foreign keys without an index. Usually you won't find this to be a problem, but it does mean that association methods that build new models will not work correctly -- for example, ```user.addresses.new``` returns an address that is not associated to the user. We'll be correcting this ~soon~ maybe someday, if we get a pull request.
 
 ### Validations
 
@@ -327,14 +337,14 @@ It also supports .gte and .lte. Turning those into symbols and allowing a Rails
 
 ## Concurrency
 
-Dynamoid supports basic, ActiveRecord-like optimistic locking on save operations. Simply add a `lock_version` column to your table like so: 
+Dynamoid supports basic, ActiveRecord-like optimistic locking on save operations. Simply add a `lock_version` column to your table like so:
 
 ```ruby
 class MyTable
   ...
-  
+
   field :lock_version, :integer
-  
+
   ...
 end
 ```
@@ -342,7 +352,49 @@ end
 In this example, all saves to `MyTable` will raise an `Dynamoid::Errors::StaleObjectError` if a concurrent process loaded, edited, and saved the same row. Your code should trap this exception, reload the row (so that it will pick up the newest values), and try the save again.
 
 Calls to `update` and `update!` also increment the `lock_version`, however they do not check the existing value. This guarantees that a update operation will raise an exception in a concurrent save operation, however a save operation will never cause an update to fail. Thus, `update` is useful & safe only for doing atomic operations (e.g. increment a value, add/remove from a set, etc), but should not be used in a read-modify-write pattern.
- 
+
+## Test Environment
+
+In test environment you will most likely want to clean the database between test runs to keep tests completely isolated. This can be achieved like so
+
+```ruby
+module DynamoidReset
+  def self.all
+    Dynamoid.adapter.list_tables.each do |table|
+      # Only delete tables in our namespace
+      if table =~ /^#{Dynamoid::Config.namespace}/
+        Dynamoid.adapter.delete_table(table)
+      end
+    end
+    Dynamoid.adapter.tables.clear
+    # Recreate all tables to avoid unexpected errors
+    Dynamoid.included_models.each(&:create_table)
+  end
+end
+
+# Reduce noise in test output
+Dynamoid.logger.level = Logger::FATAL
+```
+
+If you're using RSpec you can invoke the above like so:
+
+```ruby
+RSpec.configure do |config|
+  config.before(:each) do
+    DynamoidReset.all
+  end
+end
+```
+
+In Rails, you may also want to ensure you do not delete non-test data accidentally by adding the following to your test environment setup:
+
+```ruby
+raise "Tests should be run in 'test' environment only" if Rails.env != 'test'
+Dynamoid.configure do |config|
+  config.namespace = "#{Rails.application.railtie_name}_#{Rails.env}"
+end
+```
+
 ## Credits
 
 Dynamoid borrows code, structure, and even its name very liberally from the truly amazing [Mongoid](https://github.com/mongoid/mongoid). Without Mongoid to crib from none of this would have been possible, and I hope they don't mind me reusing their very awesome ideas to make DynamoDB just as accessible to the Ruby world as MongoDB.
@@ -363,7 +415,7 @@ Also, without contributors the project wouldn't be nearly as awesome. So many th
 * [Brian Glusman](https://github.com/bglusman) *
 * [Peter Boling](https://github.com/pboling) *
 
-\* Current Maintianers
+\* Current Maintainers
 
 ## Running the tests
 
@@ -380,9 +432,9 @@ Running the tests is fairly simple. You should have an instance of DynamoDB runn
     ```shell
     bin/start_dynamodblocal
     ```
- 
+
  * and lastly, use `rake` to run the tests.
- 
+
     ```shell
     rake
     ```

data/dynamoid.gemspec

@@ -38,7 +38,11 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency(%q<activemodel>, ["~> 4"])
+  if Gem::Version.new(RUBY_VERSION) > Gem::Version.new("2.2.2")
+    spec.add_runtime_dependency(%q<activemodel>, [">= 4"])
+  else
+    spec.add_runtime_dependency(%q<activemodel>, ["~> 4"])
+  end
   spec.add_runtime_dependency(%q<aws-sdk-resources>, ["~> 2"])
   spec.add_runtime_dependency(%q<concurrent-ruby>, [">= 1.0"])
   spec.add_development_dependency(%q<rake>, [">= 10"])
@@ -49,4 +53,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency(%q<pry>, [">= 0"])
   spec.add_development_dependency(%q<coveralls>, [">= 0"])
   spec.add_development_dependency(%q<rspec-retry>, [">= 0"])
+  spec.add_development_dependency(%q<activesupport>, ["~> 4"])
 end

data/lib/dynamoid/adapter_plugin/aws_sdk_v2.rb

@@ -36,14 +36,29 @@ module Dynamoid
       #
       # @return [Aws::DynamoDB::Client] the DynamoDB connection
       def connect!
-        @client = if Dynamoid::Config.endpoint?
-          Aws::DynamoDB::Client.new(endpoint: Dynamoid::Config.endpoint)
-        else
-          Aws::DynamoDB::Client.new
-        end
+        @client = Aws::DynamoDB::Client.new(connection_config)
         @table_cache = {}
       end
 
+      def connection_config
+        @connection_hash = {}
+
+        if Dynamoid::Config.endpoint?
+          @connection_hash[:endpoint] = Dynamoid::Config.endpoint
+        end
+        if Dynamoid::Config.access_key?
+          @connection_hash[:access_key_id] = Dynamoid::Config.access_key
+        end
+        if Dynamoid::Config.secret_key?
+          @connection_hash[:secret_access_key] = Dynamoid::Config.secret_key
+        end
+        if Dynamoid::Config.region?
+          @connection_hash[:region] = Dynamoid::Config.region
+        end
+
+        @connection_hash
+      end
+
       # Return the client object.
       #
       # @since 1.0.0
@@ -97,35 +112,41 @@ module Dynamoid
         request_items = Hash.new{|h, k| h[k] = []}
         return request_items if table_ids.all?{|k, v| v.empty?}
 
+        ret = Hash.new([].freeze) # Default for tables where no rows are returned
+
         table_ids.each do |t, ids|
           next if ids.empty?
           tbl = describe_table(t)
           hk  = tbl.hash_key.to_s
           rng = tbl.range_key.to_s
 
-          keys = if rng.present?
-            Array(ids).map do |h,r|
-              { hk => h, rng => r }
-            end
-          else
-            Array(ids).map do |id|
-              { hk => id }
+          Array(ids).each_slice(Dynamoid::Config.batch_size) do |ids|
+            request_items = Hash.new{|h, k| h[k] = []}
+
+            keys = if rng.present?
+              Array(ids).map do |h,r|
+                { hk => h, rng => r }
+              end
+            else
+              Array(ids).map do |id|
+                { hk => id }
+              end
             end
-          end
 
-          request_items[t] = {
-            keys: keys
-          }
-        end
+            request_items[t] = {
+              keys: keys
+            }
 
-        results = client.batch_get_item(
-          request_items: request_items
-        )
+            results = client.batch_get_item(
+              request_items: request_items
+            )
 
-        ret = Hash.new([].freeze) # Default for tables where no rows are returned
-        results.data[:responses].each do |table, rows|
-          ret[table] = rows.collect { |r| result_item_to_hash(r) }
+            results.data[:responses].each do |table, rows|
+              ret[table] += rows.collect { |r| result_item_to_hash(r) }
+            end
+          end
         end
+
         ret
       end
 
@@ -447,11 +468,16 @@ module Dynamoid
         q[:key_conditions] = key_conditions
 
         Enumerator.new { |y|
-          result = client.query(q)
+          loop do
+            results = client.query(q)
+            results.items.each { |row| y << result_item_to_hash(row) }
 
-          result.items.each { |r|
-            y << result_item_to_hash(r)
-          }
+            if(lk = results.last_evaluated_key)
+              q[:exclusive_start_key] = lk
+            else
+              break
+            end
+          end
         }
       end
 

data/lib/dynamoid/associations/belongs_to.rb

@@ -29,14 +29,14 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0
       def associate_target(object)
-        object.update_attribute(target_attribute, target_ids.merge(Array(source.id)))
+        object.update_attribute(target_attribute, target_ids.merge(Array(source.hash_key)))
       end
 
       # Disassociate a source object from this association.
       #
       # @since 0.2.0      
       def disassociate_target(object)
-        source.update_attribute(source_attribute, target_ids - Array(source.id))
+        source.update_attribute(source_attribute, target_ids - Array(source.hash_key))
       end
     end
   end

data/lib/dynamoid/associations/has_and_belongs_to_many.rb

@@ -24,7 +24,7 @@ module Dynamoid #:nodoc:
       # @since 0.2.0
       def associate_target(object)
         ids = object.send(target_attribute) || Set.new
-        object.update_attribute(target_attribute, ids.merge(Array(source.id)))
+        object.update_attribute(target_attribute, ids.merge(Array(source.hash_key)))
       end
 
       # Disassociate a source object from this association.
@@ -32,7 +32,7 @@ module Dynamoid #:nodoc:
       # @since 0.2.0
       def disassociate_target(object)
         ids = object.send(target_attribute) || Set.new
-        object.update_attribute(target_attribute, ids - Array(source.id))
+        object.update_attribute(target_attribute, ids - Array(source.hash_key))
       end
     end
   end

data/lib/dynamoid/associations/has_many.rb

@@ -23,7 +23,7 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0                  
       def associate_target(object)
-        object.update_attribute(target_attribute, Set[source.id])
+        object.update_attribute(target_attribute, Set[source.hash_key])
       end
       
       # Disassociate a source object from this association.

data/lib/dynamoid/associations/has_one.rb

@@ -24,7 +24,7 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0
       def associate_target(object)
-        object.update_attribute(target_attribute, Set[source.id])
+        object.update_attribute(target_attribute, Set[source.hash_key])
       end
 
       # Disassociate a source object from this association.

data/lib/dynamoid/associations/many_association.rb

@@ -52,7 +52,7 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0
       def delete(object)
-        source.update_attribute(source_attribute, source_ids - Array(object).collect(&:id))
+        source.update_attribute(source_attribute, source_ids - Array(object).collect(&:hash_key))
         Array(object).each {|o| self.send(:disassociate_target, o)} if target_association
         object
       end
@@ -67,7 +67,7 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0
       def <<(object)
-        source.update_attribute(source_attribute, source_ids.merge(Array(object).collect(&:id)))
+        source.update_attribute(source_attribute, source_ids.merge(Array(object).collect(&:hash_key)))
         Array(object).each {|o| self.send(:associate_target, o)} if target_association
         object
       end

data/lib/dynamoid/associations/single_association.rb

@@ -9,7 +9,7 @@ module Dynamoid #:nodoc:
 
       def setter(object)
         delete
-        source.update_attribute(source_attribute, Set[object.id])
+        source.update_attribute(source_attribute, Set[object.hash_key])
         self.send(:associate_target, object) if target_association
         object
       end
@@ -53,6 +53,12 @@ module Dynamoid #:nodoc:
         target.nil?
       end
 
+      def empty?
+        # This is needed to that ActiveSupport's #blank? and #present?
+        # methods work as expected for SingleAssociations.
+        target.nil?
+      end
+
       private
 
       # Find the target of the has_one association.

data/lib/dynamoid/components.rb

@@ -23,7 +23,7 @@ module Dynamoid
     include ActiveModel::Naming
     include ActiveModel::Observing if defined?(ActiveModel::Observing)
     include ActiveModel::Serializers::JSON
-    include ActiveModel::Serializers::Xml
+    include ActiveModel::Serializers::Xml if defined?(ActiveModel::Serializers::Xml)
     include Dynamoid::Fields
     include Dynamoid::Indexes
     include Dynamoid::Persistence

data/lib/dynamoid/config.rb

@@ -14,8 +14,10 @@ module Dynamoid
     option :adapter, :default => 'aws_sdk_v2'
     option :namespace, :default => defined?(Rails) ? "dynamoid_#{Rails.application.class.parent_name}_#{Rails.env}" : "dynamoid"
     option :logger, :default => defined?(Rails)
-    option :access_key
-    option :secret_key
+    option :access_key, :default => nil
+    option :secret_key, :default => nil
+    option :region, :default => nil
+    option :batch_size, :default => 100
     option :read_capacity, :default => 100
     option :write_capacity, :default => 20
     option :warn_on_scan, :default => true
@@ -26,6 +28,7 @@ module Dynamoid
     option :timestamps, :default => true
     option :sync_retry_max_times, :default => 60 # a bit over 2 minutes
     option :sync_retry_wait_seconds, :default => 2
+    option :convert_big_decimal, :default => false
 
     # The default logger for Dynamoid: either the Rails logger or just stdout.
     #

data/lib/dynamoid/errors.rb

@@ -53,8 +53,11 @@ module Dynamoid
     end
 
     class DocumentNotValid < Error
+      attr_reader :document
+
       def initialize(document)
         super("Validation failed: #{document.errors.full_messages.join(", ")}")
+        @document = document
       end
     end
 

data/lib/dynamoid/persistence.rb

@@ -18,13 +18,8 @@ module Dynamoid
       def table_name
         table_base_name = options[:name] || base_class.name.split('::').last
           .downcase.pluralize
-        table_prefix = if Dynamoid::Config.namespace.nil? then
-          ''
-        else
-          "#{Dynamoid::Config.namespace}_"
-        end
 
-        @table_name ||= "#{table_prefix}#{table_base_name}"
+        @table_name ||= [Dynamoid::Config.namespace.to_s,table_base_name].reject(&:empty?).join("_")
       end
 
       # Creates a table.
@@ -96,7 +91,7 @@ module Dynamoid
             value = default_value.respond_to?(:call) ? default_value.call : default_value
           end
 
-          if !value.nil?
+          unless value.nil?
             case options[:type]
               when :string
                 value.to_s
@@ -106,6 +101,12 @@ module Dynamoid
                 BigDecimal.new(value.to_s)
               when :array
                 value.to_a
+              when :raw
+                if value.is_a?(Hash)
+                  undump_hash(value)
+                else
+                  value
+                end
               when :set
                 Set.new(value)
               when :datetime
@@ -145,6 +146,30 @@ module Dynamoid
         end
       end
 
+      private
+
+      def undump_hash(hash)
+        {}.tap do |h|
+          hash.each { |key, value| h[key.to_sym] = undump_hash_value(value) }
+        end
+      end
+
+      def undump_hash_value(val)
+        case val
+        when BigDecimal
+          if Dynamoid::Config.convert_big_decimal
+            val.to_f
+          else
+            val
+          end
+        when Hash
+          undump_hash(val)
+        when Array
+          val.map { |v| undump_hash_value(v) }
+        else
+          val
+        end
+      end
     end
 
     # Set updated_at and any passed in field to current DateTime. Useful for things like last_login_at, etc.
@@ -286,6 +311,8 @@ module Dynamoid
             !value.nil? ? value.to_time.to_f : nil
           when :serialized
             options[:serializer] ? options[:serializer].dump(value) : value.to_yaml
+          when :raw
+            !value.nil? ? value : nil
           when :boolean
             !value.nil? ? value.to_s[0] : nil
           else

data/lib/dynamoid/version.rb

@@ -1,3 +1,3 @@
 module Dynamoid
-  VERSION = "1.2.1"
+  VERSION = "1.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamoid
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Josh Symonds
@@ -20,7 +20,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-04 00:00:00.000000000 Z
+date: 2016-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -176,6 +176,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
 description: Dynamoid is an ORM for Amazon's DynamoDB that supports offline development,
   associations, querying, and everything else you'd expect from an ActiveRecord-style
   replacement.