dynamoid 0.1.1 → 0.1.2

data/Dynamoid.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "dynamoid"
-  s.version = "0.1.1"
+  s.version = "0.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Josh Symonds"]
-  s.date = "2012-02-27"
+  s.date = "2012-03-05"
   s.description = "Dynamoid is an ORM for Amazon's DynamoDB that supports offline development, associations, querying, and everything else you'd expect from an ActiveRecord-style replacement."
   s.email = "josh@joshsymonds.com"
   s.extra_rdoc_files = [
@@ -36,7 +36,6 @@ Gem::Specification.new do |s|
     "lib/dynamoid/associations/has_and_belongs_to_many.rb",
     "lib/dynamoid/associations/has_many.rb",
     "lib/dynamoid/associations/has_one.rb",
-    "lib/dynamoid/attributes.rb",
     "lib/dynamoid/components.rb",
     "lib/dynamoid/config.rb",
     "lib/dynamoid/config/options.rb",
@@ -48,7 +47,6 @@ Gem::Specification.new do |s|
     "lib/dynamoid/finders.rb",
     "lib/dynamoid/indexes.rb",
     "lib/dynamoid/persistence.rb",
-    "lib/dynamoid/relations.rb",
     "spec/app/models/address.rb",
     "spec/app/models/magazine.rb",
     "spec/app/models/sponsor.rb",
@@ -63,7 +61,6 @@ Gem::Specification.new do |s|
     "spec/dynamoid/associations/has_many_spec.rb",
     "spec/dynamoid/associations/has_one_spec.rb",
     "spec/dynamoid/associations_spec.rb",
-    "spec/dynamoid/attributes_spec.rb",
     "spec/dynamoid/config_spec.rb",
     "spec/dynamoid/criteria/chain_spec.rb",
     "spec/dynamoid/criteria_spec.rb",
@@ -72,7 +69,6 @@ Gem::Specification.new do |s|
     "spec/dynamoid/finders_spec.rb",
     "spec/dynamoid/indexes_spec.rb",
     "spec/dynamoid/persistence_spec.rb",
-    "spec/dynamoid/relations_spec.rb",
     "spec/dynamoid_spec.rb",
     "spec/spec_helper.rb"
   ]

data/README.markdown

@@ -2,6 +2,10 @@
 
 Dynamoid is an ORM for Amazon's DynamoDB for Ruby applications. It provides similar functionality to ActiveRecord and improves on Amazon's existing [HashModel](http://docs.amazonwebservices.com/AWSRubySDK/latest/AWS/Record/HashModel.html) by providing better searching tools, native association support, and a local adapter for offline development.
 
+DynamoDB is not like other document-based databases you might know, and is very different indeed from relational databases. It sacrifices anything beyond the simplest relational queries and transactional support to provide a fast, cost-efficient, and highly durable storage solution. If your database requires complicated relational queries and transaction support, then this modest Gem cannot provide them for you, and neither can DynamoDB. In those cases you would do better to look elsewhere for your database needs.
+
+But if you want a fast, scalable, simple, easy-to-use database (and a Gem that supports it) then look no further!
+
 ## Warning!
 
 I'm still working on this gem a lot. It only provides .where(arguments) in its criteria chaining so far. More is coming though!
@@ -18,12 +22,14 @@ Then you need to initialize it to get it going. Put code similar to this somewhe
 
 ```ruby
   Dynamoid.configure do |config|
-    config.adapter = 'local' # This adapter allows offline development without connecting to the DynamoDB servers. Data is NOT persisted.
-    # config.adapter = 'aws_sdk' # This adapter establishes a connection to the DynamoDB servers using's Amazon's own awful AWS gem.
+    config.adapter = 'local' # This adapter allows offline development without connecting to the DynamoDB servers. Data is *NOT* persisted.
+    # config.adapter = 'aws_sdk' # This adapter establishes a connection to the DynamoDB servers using Amazon's own AWS gem.
     # config.access_key = 'access_key' # If connecting to DynamoDB, your access key is required.
     # config.secret_key = 'secret_key' # So is your secret key. 
     config.namespace = "dynamoid_#{Rails.application.class.parent_name}_#{Rails.env}" # To namespace tables created by Dynamoid from other tables you might have.
-    config.warn_on_scan = true # Output a warning to stdout when you perform a scan rather than a query on a table
+    config.warn_on_scan = true # Output a warning to the logger when you perform a scan rather than a query on a table.
+    config.partitioning = true # Spread writes randomly across the database. See "partitioning" below for more.
+    config.partition_size = 200  # Determine the key space size that writes are randomly spread across.
   end
 
 ```
@@ -36,9 +42,11 @@ class User
    
    field :name           # Every field you have on the object must be specified here.
    field :email          # If you have fields that aren't specified they won't be attached to the object as methods.
+   field :rank, :integer # Every field is assumed to be a string unless otherwise specified.
+                         # created_at and updated_at with a type of :datetime are automatically added.
    
    index :name           # Only specify indexes if you intend to perform queries on the specified fields.
-   index :email          # Fields without indexes enjoy extremely poor performance as they must use 
+   index :email          # Fields without indexes suffer extremely poor performance as they must use 
    index [:name, :email] # scan rather than query.
    
    has_many :addresses   # Associations do not accept any options presently. The referenced
@@ -84,6 +92,22 @@ Address.where(:city => 'Chicago').all # Find by any number of matching criteria.
 Address.find_by_city('Chicago')       # The same as above, but using ActiveRecord's older syntax.
 ```
 
+And you can also query on associations:
+
+```ruby
+u.addresses.where(:city => 'Chicago').all
+```
+
+But keep in mind Dynamoid -- and document-based storage systems in general -- are not drop-in replacements for existing relational databases. The above query does not efficiently perform a conditional join, but instead finds all the user's addresses and naively filters them in Ruby. For large associations this is a performance hit compared to relational database engines.
+
+## Partitioning, Provisioning, and Performance
+
+DynamoDB achieves much of its speed by relying on a random pattern of writes and reads: internally, hash keys are distributed across servers, and reading from two consecutive servers is much faster than reading from the same server twice. Of course, many of our applications request one key (like a commonly used role, a superuser, or a very popular product) much more frequently than other keys. In DynamoDB, this will result in lowered throughput and slower response times, and is a design pattern we should try to avoid.
+
+Dynamoid attempts to obviate this problem transparently by employing a partitioning strategy to divide up keys randomly across DynamoDB's servers. Each ID is assigned an additional number (by default 0 to 199, but you can increase the partition size in Dynamoid's configuration) upon save; when read, all 200 hashes are retrieved simultaneously and the most recently updated one is returned to the application. This results in a significant net performance increase, and is usually invisible to the application itself. It does, however, bring up the important issue of provisioning your DynamoDB tables correctly.
+
+When your read or write provisioning exceed your table's allowed throughput, DynamoDB will wait on connections until throughput is available again. This will appear as very, very slow requests and can be somewhat frustrating. Partitioning significantly increases the amount of throughput tables will experience; though DynamoDB will ignore keys that don't exist, if you have 20 partitioned keys representing one object, all will be retrieved every time the object is requested. Ensure that your tables are set up for this kind of throughput, or turn provisioning off, to make sure that DynamoDB doesn't throttle your requests.
+
 ## Credits
 
 Dynamoid borrows code, structure, and even its name very liberally from the truly amazing [Mongoid](https://github.com/mongoid/mongoid). Without Mongoid to crib from none of this would have been possible, and I hope they don't mind me reusing their very awesome ideas to make DynamoDB just as accessible to the Ruby world as MongoDB.

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/lib/dynamoid.rb

@@ -9,7 +9,6 @@ require "active_support/time_with_zone"
 require "active_model"
 
 require 'dynamoid/errors'
-require 'dynamoid/attributes'
 require 'dynamoid/fields'
 require 'dynamoid/indexes'
 require 'dynamoid/associations'
@@ -34,4 +33,8 @@ module Dynamoid
     Dynamoid::Config.logger
   end
   
+  def included_models
+    @included_models ||= []
+  end
+  
 end

data/lib/dynamoid/adapter.rb

@@ -3,6 +3,7 @@ module Dynamoid #:nodoc:
 
   module Adapter
     extend self
+    attr_accessor :tables
     
     def adapter
       reconnect! unless @adapter
@@ -13,17 +14,89 @@ module Dynamoid #:nodoc:
       require "dynamoid/adapter/#{Dynamoid::Config.adapter}" unless Dynamoid::Adapter.const_defined?(Dynamoid::Config.adapter.camelcase)
       @adapter = Dynamoid::Adapter.const_get(Dynamoid::Config.adapter.camelcase)
       @adapter.connect! if @adapter.respond_to?(:connect!)
+      self.tables = benchmark('Cache Tables') {list_tables}
     end
     
-    def method_missing(method, *args)
-      if @adapter.respond_to?(method)
-        start = Time.now
-        result = @adapter.send(method, *args)
-        Dynamoid.logger.info "(#{((Time.now - start) * 1000.0).round(2)} ms) #{method.to_s.split('_').collect(&:upcase).join(' ')}#{ " - #{args.join(',')}" unless args.empty? }"
-        return result
+    def benchmark(method, *args)
+      start = Time.now
+      result = yield
+      Dynamoid.logger.info "(#{((Time.now - start) * 1000.0).round(2)} ms) #{method.to_s.split('_').collect(&:upcase).join(' ')}#{ " - #{args.inspect}" unless args.nil? || args.empty? }"
+      return result
+    end
+    
+    def write(table, object)
+      if Dynamoid::Config.partitioning? && object[:id]
+        object[:id] = "#{object[:id]}.#{Random.rand(Dynamoid::Config.partition_size)}"
+        object[:updated_at] = Time.now.to_f
+      end
+      benchmark('Put Item', object) {put_item(table, object)}
+    end
+    
+    def read(table, ids)
+      if ids.respond_to?(:each)
+        if Dynamoid::Config.partitioning?
+          results = benchmark('Partitioned Batch Get Item', ids) {batch_get_item(table => id_with_partitions(ids))}
+          {table => result_for_partition(results[table])}
+        else
+          benchmark('Batch Get Item', ids) {batch_get_item(table => ids)}
+        end
+      else
+        if Dynamoid::Config.partitioning?
+          results = benchmark('Partitioned Get Item', ids) {batch_get_item(table => id_with_partitions(ids))}
+          result_for_partition(results[table]).first
+        else
+          benchmark('Get Item', ids) {get_item(table, ids)}
+        end
+      end
+    end
+    
+    def delete(table, id)
+      if Dynamoid::Config.partitioning?
+        benchmark('Delete Item', id) do
+          id_with_partitions(id).each {|i| delete_item(table, i)}
+        end
+      else
+        benchmark('Delete Item', id) {delete_item(table, id)}
+      end
+    end
+    
+    def scan(table, query)
+      if Dynamoid::Config.partitioning?
+        results = benchmark('Scan', table, query) {adapter.scan(table, query)}
+        result_for_partition(results)
+      else
+        adapter.scan(table, query)
       end
+    end
+    
+    [:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item, :query].each do |m|
+      define_method(m) do |*args|
+        benchmark("#{m.to_s}", args) {adapter.send(m, *args)}
+      end
+    end
+    
+    def id_with_partitions(ids)
+      Array(ids).collect {|id| (0...Dynamoid::Config.partition_size).collect{|n| "#{id}.#{n}"}}.flatten
+    end
+    
+    def result_for_partition(results)
+      Hash.new.tap do |hash|
+        Array(results).each do |result|
+          next if result.nil?
+          id = result[:id].split('.').first
+          if !hash[id] || (result[:updated_at] > hash[id][:updated_at])
+            result[:id] = id
+            hash[id] = result
+          end
+        end
+      end.values
+    end
+    
+    def method_missing(method, *args)
+      return benchmark(method, *args) {adapter.send(method, *args)} if @adapter.respond_to?(method)
       super
     end
+    
   end
   
 end

data/lib/dynamoid/adapter/aws_sdk.rb

@@ -16,21 +16,23 @@ module Dynamoid
     
       # BatchGetItem
       def batch_get_item(options)
-        batch = AWS::DynamoDB::BatchGet.new(:config => @@connection.config)
         hash = Hash.new{|h, k| h[k] = []}
         return hash if options.all?{|k, v| v.empty?}
         options.each do |t, ids|
-          batch.table(t, :all, Array(ids)) unless ids.nil? || ids.empty?
-        end
-        batch.each do |table_name, attributes|
-          hash[table_name] << attributes.symbolize_keys!
+          Array(ids).in_groups_of(100, false) do |group|
+            batch = AWS::DynamoDB::BatchGet.new(:config => @@connection.config)
+            batch.table(t, :all, Array(group)) unless group.nil? || group.empty?
+            batch.each do |table_name, attributes|
+              hash[table_name] << attributes.symbolize_keys!
+            end
+          end
         end
         hash
       end
     
       # CreateTable
       def create_table(table_name, key)
-        table = @@connection.tables.create(table_name, 10, 5, :hash_key => {key.to_sym => :string})
+        table = @@connection.tables.create(table_name, 100, 20, :hash_key => {key.to_sym => :string})
         sleep 0.5 while table.status == :creating
         return table
       end
@@ -72,7 +74,7 @@ module Dynamoid
       def put_item(table_name, object)
         table = @@connection.tables[table_name]
         table.load_schema
-        table.items.create(object.delete_if{|k, v| v.nil? || v.empty?})
+        table.items.create(object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)})
       end
     
       # Query

data/lib/dynamoid/adapter/local.rb

@@ -54,7 +54,7 @@ module Dynamoid
       # PutItem
       def put_item(table_name, object)
         table = data[table_name]
-        table[:data][object[table[:id]]] = object
+        table[:data][object[table[:id]]] = object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)}
       end
     
       # Query

data/lib/dynamoid/associations.rb

@@ -37,7 +37,7 @@ module Dynamoid #:nodoc:
       private
       
       def association(type, name, options = {})
-        field "#{name}_ids".to_sym
+        field "#{name}_ids".to_sym, :set
         self.associations[name] = options.merge(:type => type)
         define_method(name) do
           @associations ||= {}

data/lib/dynamoid/associations/association.rb

@@ -4,14 +4,24 @@ module Dynamoid #:nodoc:
   # The base association module.
   module Associations
     module Association
-      attr_accessor :name, :options, :source
+      attr_accessor :name, :options, :source, :query
+      include Enumerable
 
       def initialize(source, name, options)
         @name = name
         @options = options
         @source = source
+        @query = {}
       end
       
+      def records
+        results = target_class.find(source_ids.to_a)
+        results = results.nil? ? [] : Array(results)
+        return results if query.empty?
+        results_with_query(results)
+      end
+      alias :all :records
+      
       def empty?
         records.empty?
       end
@@ -49,11 +59,23 @@ module Dynamoid #:nodoc:
         self << object
       end
       
+      def where(args)
+        args.each {|k, v| query[k] = v}
+        self
+      end
+      
+      def each(&block)
+        records.each(&block)
+      end
+      
       private
       
-      def records
-        results = target_class.find(source_ids.to_a)
-        results.nil? ? [] : Array(results)
+      def results_with_query(results)
+        results.find_all do |result|
+          query.all? do |attribute, value|
+            result.send(attribute) == value
+          end
+        end
       end
       
       def target_class

data/lib/dynamoid/associations/belongs_to.rb

@@ -10,6 +10,14 @@ module Dynamoid #:nodoc:
         target == other
       end
       
+      def method_missing(method, *args)
+        if target.respond_to?(method)
+          target.send(method, *args)
+        else
+          super
+        end
+      end
+      
       private
       
       def target

data/lib/dynamoid/components.rb

@@ -10,6 +10,9 @@ module Dynamoid #:nodoc
       extend ActiveModel::Callbacks
 
       define_model_callbacks :create, :save, :destroy
+      
+      before_create :set_created_at
+      before_save :set_updated_at
     end
 
     include ActiveModel::Conversion
@@ -19,7 +22,6 @@ module Dynamoid #:nodoc
     include ActiveModel::Validations
     include ActiveModel::Serializers::JSON
     include ActiveModel::Serializers::Xml
-    include Dynamoid::Attributes
     include Dynamoid::Fields
     include Dynamoid::Indexes
     include Dynamoid::Persistence

data/lib/dynamoid/config.rb

@@ -15,6 +15,9 @@ module Dynamoid #:nodoc
     option :access_key
     option :secret_key
     option :warn_on_scan, :default => true
+    option :partitioning, :default => false
+    option :partition_size, :default => 200
+    option :included_models, :default => []
     
     def default_logger
       defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : ::Logger.new($stdout)

data/lib/dynamoid/criteria/chain.rb

@@ -38,7 +38,7 @@ module Dynamoid #:nodoc:
       end
       
       def records_with_index
-        ids = Dynamoid::Adapter.get_item(source.index_table_name(index), source.key_for_index(index, values_for_index))
+        ids = Dynamoid::Adapter.read(source.index_table_name(index), source.key_for_index(index, values_for_index))
         if ids.nil? || ids.empty?
           []
         else

data/lib/dynamoid/document.rb

@@ -6,22 +6,9 @@ module Dynamoid #:nodoc:
   module Document
     extend ActiveSupport::Concern
     include Dynamoid::Components
-
-    attr_accessor :new_record
-    alias :new_record? :new_record
-    
-    def initialize(attrs = {})
-      @new_record = true
-      @attributes ||= {}
-      self.class.attributes.each {|att| write_attribute(att, attrs[att])}
-    end
     
-    def persisted?
-      !new_record?
-    end
-    
-    def ==(other)
-      other.respond_to?(:id) && other.id == self.id
+    included do
+      Dynamoid::Config.included_models << self
     end
     
     module ClassMethods
@@ -37,6 +24,22 @@ module Dynamoid #:nodoc:
         self.new(attrs)
       end
     end
+    
+    def initialize(attrs = {})
+      @new_record = true
+      @attributes ||= {}
+      attrs = self.class.undump(attrs)
+      self.class.attributes.keys.each {|att| write_attribute(att, attrs[att])}
+    end
+    
+    def ==(other)
+      other.respond_to?(:id) && other.id == self.id
+    end
+    
+    def reload
+      self.attributes = self.class.find(self.id).attributes
+      self
+    end
   end
   
 end

data/lib/dynamoid/fields.rb

@@ -5,16 +5,18 @@ module Dynamoid #:nodoc:
     extend ActiveSupport::Concern
 
     included do
-      class_attribute :fields
+      class_attribute :attributes
       
-      self.fields = []
+      self.attributes = {}
       field :id
+      field :created_at, :datetime
+      field :updated_at, :datetime
     end
     
     module ClassMethods
-      def field(name, options = {})
+      def field(name, type = :string, options = {})
         named = name.to_s
-        self.fields << name
+        self.attributes[name] = {:type => type}.merge(options)
         define_method(named) do
           read_attribute(named)
         end
@@ -27,6 +29,39 @@ module Dynamoid #:nodoc:
       end
     end
     
+    attr_accessor :attributes
+    alias :raw_attributes :attributes
+
+    def write_attribute(name, value)
+      attributes[name.to_sym] = value
+    end
+    alias :[]= :write_attribute
+
+    def read_attribute(name)
+      attributes[name.to_sym]
+    end
+    alias :[] :read_attribute
+
+    def update_attributes(attributes)
+      attributes.each {|attribute, value| self.write_attribute(attribute, value)}
+      save
+    end
+
+    def update_attribute(attribute, value)
+      write_attribute(attribute, value)
+      save
+    end
+    
+    private
+    
+    def set_created_at
+      self.created_at = DateTime.now
+    end
+    
+    def set_updated_at
+      self.updated_at = DateTime.now
+    end
+    
   end
   
 end