dynamoid 0.1.2 → 0.2.0

data/Dynamoid.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "dynamoid"
-  s.version = "0.1.2"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Josh Symonds"]
-  s.date = "2012-03-05"
+  s.date = "2012-03-15"
   s.description = "Dynamoid is an ORM for Amazon's DynamoDB that supports offline development, associations, querying, and everything else you'd expect from an ActiveRecord-style replacement."
   s.email = "josh@joshsymonds.com"
   s.extra_rdoc_files = [
@@ -46,6 +46,7 @@ Gem::Specification.new do |s|
     "lib/dynamoid/fields.rb",
     "lib/dynamoid/finders.rb",
     "lib/dynamoid/indexes.rb",
+    "lib/dynamoid/indexes/index.rb",
     "lib/dynamoid/persistence.rb",
     "spec/app/models/address.rb",
     "spec/app/models/magazine.rb",
@@ -67,6 +68,7 @@ Gem::Specification.new do |s|
     "spec/dynamoid/document_spec.rb",
     "spec/dynamoid/fields_spec.rb",
     "spec/dynamoid/finders_spec.rb",
+    "spec/dynamoid/indexes/index_spec.rb",
     "spec/dynamoid/indexes_spec.rb",
     "spec/dynamoid/persistence_spec.rb",
     "spec/dynamoid_spec.rb",
@@ -75,7 +77,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/Veraticus/Dynamoid"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
+  s.rubygems_version = "1.8.18"
   s.summary = "Dynamoid is an ORM for Amazon's DynamoDB"
 
   if s.respond_to? :specification_version then

data/README.markdown

@@ -6,10 +6,6 @@ DynamoDB is not like other document-based databases you might know, and is very
 
 But if you want a fast, scalable, simple, easy-to-use database (and a Gem that supports it) then look no further!
 
-## Warning!
-
-I'm still working on this gem a lot. It only provides .where(arguments) in its criteria chaining so far. More is coming though!
-
 ## Installation
 
 Installing Dynamoid is pretty simple. First include the Gem in your Gemfile:
@@ -48,6 +44,10 @@ class User
    index :name           # Only specify indexes if you intend to perform queries on the specified fields.
    index :email          # Fields without indexes suffer extremely poor performance as they must use 
    index [:name, :email] # scan rather than query.
+   index :created_at, :range => true
+   index :name, :range => :created_at
+                         # You can only provide one range query for each index, or specify an index
+                         # to be only a range query with :range => true.
    
    has_many :addresses   # Associations do not accept any options presently. The referenced
                          # model name must match exactly and the foreign key is always id.
@@ -100,6 +100,15 @@ u.addresses.where(:city => 'Chicago').all
 
 But keep in mind Dynamoid -- and document-based storage systems in general -- are not drop-in replacements for existing relational databases. The above query does not efficiently perform a conditional join, but instead finds all the user's addresses and naively filters them in Ruby. For large associations this is a performance hit compared to relational database engines.
 
+If you have a range index, Dynamoid provides a number of additional other convenience methods to make your life a little easier:
+
+```ruby
+User.where("created_at.gt" => DateTime.now - 1.day).all
+User.where("created_at.lt" => DateTime.now - 1.day).all
+```
+
+It also supports .gte and .lte. Turning those into symbols and allowing a Rails SQL-style string syntax is in the works. You can only have one range argument per query, because of DynamoDB's inherent limitations, so use it sensibly!
+
 ## Partitioning, Provisioning, and Performance
 
 DynamoDB achieves much of its speed by relying on a random pattern of writes and reads: internally, hash keys are distributed across servers, and reading from two consecutive servers is much faster than reading from the same server twice. Of course, many of our applications request one key (like a commonly used role, a superuser, or a very popular product) much more frequently than other keys. In DynamoDB, this will result in lowered throughput and slower response times, and is a design pattern we should try to avoid.

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.2.0

data/lib/dynamoid/adapter.rb

@@ -32,8 +32,9 @@ module Dynamoid #:nodoc:
       benchmark('Put Item', object) {put_item(table, object)}
     end
     
-    def read(table, ids)
+    def read(table, ids, range_key = nil)
       if ids.respond_to?(:each)
+        ids = ids.collect{|id| range_key ? [id, range_key] : id}
         if Dynamoid::Config.partitioning?
           results = benchmark('Partitioned Batch Get Item', ids) {batch_get_item(table => id_with_partitions(ids))}
           {table => result_for_partition(results[table])}
@@ -42,10 +43,11 @@ module Dynamoid #:nodoc:
         end
       else
         if Dynamoid::Config.partitioning?
+          ids = range_key ? [[ids, range_key]] : ids
           results = benchmark('Partitioned Get Item', ids) {batch_get_item(table => id_with_partitions(ids))}
           result_for_partition(results[table]).first
         else
-          benchmark('Get Item', ids) {get_item(table, ids)}
+          benchmark('Get Item', ids) {get_item(table, ids, range_key)}
         end
       end
     end
@@ -69,14 +71,14 @@ module Dynamoid #:nodoc:
       end
     end
     
-    [:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item, :query].each do |m|
+    [:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item].each do |m|
       define_method(m) do |*args|
         benchmark("#{m.to_s}", args) {adapter.send(m, *args)}
       end
     end
     
     def id_with_partitions(ids)
-      Array(ids).collect {|id| (0...Dynamoid::Config.partition_size).collect{|n| "#{id}.#{n}"}}.flatten
+      Array(ids).collect {|id| (0...Dynamoid::Config.partition_size).collect{|n| id.is_a?(Array) ? ["#{id.first}.#{n}", id.last] : "#{id}.#{n}"}}.flatten(1)
     end
     
     def result_for_partition(results)

data/lib/dynamoid/adapter/aws_sdk.rb

@@ -31,17 +31,23 @@ module Dynamoid
       end
     
       # CreateTable
-      def create_table(table_name, key)
-        table = @@connection.tables.create(table_name, 100, 20, :hash_key => {key.to_sym => :string})
+      def create_table(table_name, key = :id, options = {})
+        options[:hash_key] ||= {key.to_sym => :string}
+        options[:range_key] = {options[:range_key].to_sym => :number} if options[:range_key]
+        table = @@connection.tables.create(table_name, 100, 20, options)
         sleep 0.5 while table.status == :creating
         return table
       end
     
       # DeleteItem
-      def delete_item(table_name, key)
+      def delete_item(table_name, key, range_key = nil)
         table = @@connection.tables[table_name]
         table.load_schema
-        result = table.items[key]
+        result = if table.composite_key?
+          table.items.at(key, range_key)
+        else
+          table.items[key]
+        end
         result.delete unless result.attributes.to_h.empty?
         true
       end
@@ -54,15 +60,21 @@ module Dynamoid
       # DescribeTable
     
       # GetItem
-      def get_item(table_name, key)
+      def get_item(table_name, key, range_key = nil)
         table = @@connection.tables[table_name]
         table.load_schema
-        result = table.items[key].attributes.to_h
+        result = if table.composite_key?
+          table.items.at(key, range_key)
+        else
+          table.items[key]
+        end.attributes.to_h
         if result.empty?
           nil
         else
           result.symbolize_keys!
         end
+      rescue
+        raise [table_name, key, range_key].inspect
       end
     
       # ListTables
@@ -78,8 +90,17 @@ module Dynamoid
       end
     
       # Query
-      def query(table_name, id)
-        get_item(table_name, id)
+      def query(table_name, opts = {})
+        table = @@connection.tables[table_name]
+        table.load_schema
+        
+        if table.composite_key?
+          results = []
+          table.items.query(opts).each {|data| results << data.attributes.to_h.symbolize_keys!}
+          results
+        else
+          get_item(table_name, opts[:hash_value])
+        end
       end
     
       # Scan

data/lib/dynamoid/adapter/local.rb

@@ -17,21 +17,27 @@ module Dynamoid
         Hash.new { |h, k| h[k] = Array.new }.tap do |hash|
           options.each do |table_name, keys|
             table = data[table_name]
-            Array(keys).each do |key|
-              hash[table_name] << table[:data][key]
+            if table[:range_key]
+              Array(keys).each do |hash_key, range_key|
+                hash[table_name] << get_item(table_name, hash_key, range_key)
+              end
+            else
+              Array(keys).each do |key|
+                hash[table_name] << get_item(table_name, key)
+              end
             end
           end
         end
       end
     
       # CreateTable
-      def create_table(table_name, key)
-        data[table_name] = {:id => key, :data => {}}
+      def create_table(table_name, key, options = {})
+        data[table_name] = {:hash_key => key, :range_key => options[:range_key], :data => {}}
       end
     
       # DeleteItem
-      def delete_item(table_name, key)
-        data[table_name][:data].delete(key)
+      def delete_item(table_name, key, range_key = nil)
+        data[table_name][:data].delete("#{key}.#{range_key}")
       end
     
       # DeleteTable
@@ -42,8 +48,12 @@ module Dynamoid
       # DescribeTable
     
       # GetItem
-      def get_item(table_name, key)
-        data[table_name][:data][key]
+      def get_item(table_name, key, range_key = nil)
+        if data[table_name][:data]
+          data[table_name][:data]["#{key}.#{range_key}"]
+        else
+          nil
+        end
       end
     
       # ListTables
@@ -54,18 +64,33 @@ module Dynamoid
       # PutItem
       def put_item(table_name, object)
         table = data[table_name]
-        table[:data][object[table[:id]]] = object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)}
+        table[:data][object[table[:hash_key]]]
+        table[:data]["#{object[table[:hash_key]]}.#{object[table[:range_key]]}"] = object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)}
       end
     
       # Query
-      def query(table_name, id)
-        get_item(table_name, id)
+      def query(table_name, opts = {})
+        id = opts[:hash_value]
+        range_key = data[table_name][:range_key]
+        if opts[:range_value]
+          data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && opts[:range_value].include?(v[range_key])}
+        elsif opts[:range_greater_than]
+          data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] > opts[:range_greater_than]}
+        elsif opts[:range_less_than]
+          data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] < opts[:range_less_than]}
+        elsif opts[:range_gte]
+          data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] >= opts[:range_gte]}
+        elsif opts[:range_lte]  
+          data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] <= opts[:range_lte]}
+        else
+          get_item(table_name, id)
+        end
       end
     
       # Scan
       def scan(table_name, scan_hash)
         return [] if data[table_name].nil?
-        data[table_name][:data].values.select{|d| scan_hash.all?{|k, v| !d[k].nil? && d[k] == v}}
+        data[table_name][:data].values.flatten.select{|d| scan_hash.all?{|k, v| !d[k].nil? && d[k] == v}}
       end
     
       # UpdateItem

data/lib/dynamoid/associations/has_one.rb

@@ -9,6 +9,14 @@ module Dynamoid #:nodoc:
       def ==(other)
         target == other
       end
+
+      def method_missing(method, *args)
+        if target.respond_to?(method)
+          target.send(method, *args)
+        else
+          super
+        end
+      end
       
       private
       
@@ -17,7 +25,7 @@ module Dynamoid #:nodoc:
       end
       
       def target_association
-        key_name = source.class.to_s.pluralize.downcase.to_sym
+        key_name = source.class.to_s.singularize.downcase.to_sym
         guess = target_class.associations[key_name]
         return nil if guess.nil? || guess[:type] != :belongs_to
         key_name
@@ -33,4 +41,4 @@ module Dynamoid #:nodoc:
     end
   end
   
-end
+end

data/lib/dynamoid/criteria.rb

@@ -1,6 +1,6 @@
+# encoding: utf-8
 require 'dynamoid/criteria/chain'
 
-# encoding: utf-8
 module Dynamoid #:nodoc:
 
   # This module defines criteria and criteria chains.

data/lib/dynamoid/criteria/chain.rb

@@ -33,16 +33,25 @@ module Dynamoid #:nodoc:
       private
       
       def records
-        return records_with_index unless index.empty?
+        return records_with_index if index
         records_without_index
       end
       
       def records_with_index
-        ids = Dynamoid::Adapter.read(source.index_table_name(index), source.key_for_index(index, values_for_index))
+        ids = if index.range_key?
+          Dynamoid::Adapter.query(index.table_name, index_query).collect{|r| r[:ids]}.inject(Set.new) {|set, result| set + result}
+        else
+          results = Dynamoid::Adapter.read(index.table_name, index_query[:hash_value])
+          if results
+            results[:ids]
+          else
+            []
+          end
+        end
         if ids.nil? || ids.empty?
           []
         else
-          Array(source.find(ids[:ids].to_a))
+          Array(source.find(ids.to_a))
         end
       end
       
@@ -54,12 +63,39 @@ module Dynamoid #:nodoc:
         Dynamoid::Adapter.scan(source.table_name, query).collect {|hash| source.new(hash)}
       end
       
-      def values_for_index
-        [].tap {|arr| index.each{|i| arr << query[i]}}
+      def index_query
+        values = index.values(query)
+        {}.tap do |hash|
+          hash[:hash_value] = values[:hash_value]
+          if index.range_key?
+            key = query.keys.find{|k| k.to_s.include?('.')}
+            if key
+              if query[key].is_a?(Range)
+                hash[:range_value] = query[key]
+              else
+                val = query[key].to_f
+                case key.split('.').last
+                when 'gt'
+                  hash[:range_greater_than] = val
+                when 'lt'
+                  hash[:range_less_than] = val
+                when 'gte'
+                  hash[:range_gte] = val
+                when 'lte'
+                  hash[:range_lte] = val
+                end
+              end
+            else
+              raise Dynamoid::Errors::MissingRangeKey, 'This index requires a range key'
+            end
+          end
+        end
       end
       
       def index
-        Array(source.indexes.find {|i| i == query.keys.sort.collect(&:to_sym)})
+        index = source.find_index(query.keys.collect{|k| k.to_s.split('.').first})
+        return nil if index.blank?
+        index
       end
     end
     

data/lib/dynamoid/errors.rb

@@ -3,6 +3,6 @@ module Dynamoid
   module Errors
 
     class InvalidField < Exception; end
-    
+    class MissingRangeKey < Exception; end
   end
 end

data/lib/dynamoid/indexes.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+require 'dynamoid/indexes/index'
+
 module Dynamoid #:nodoc:
 
   # Builds all indexes present on the model.
@@ -8,53 +10,39 @@ module Dynamoid #:nodoc:
     included do
       class_attribute :indexes
       
-      self.indexes = []
+      self.indexes = {}
     end
     
     module ClassMethods
       def index(name, options = {})
-        name = Array(name).collect(&:to_s).sort.collect(&:to_sym)
-        raise Dynamoid::Errors::InvalidField, 'A key specified for an index is not a field' unless name.all?{|n| self.attributes.include?(n)}
-        self.indexes << name
-        create_indexes
-      end
-      
-      def create_indexes
-        self.indexes.each do |index|
-          self.create_table(index_table_name(index), :id) unless self.table_exists?(index_table_name(index))
-        end
+        index = Dynamoid::Indexes::Index.new(self, name, options)
+        self.indexes[index.name] = index
+        create_indexes        
       end
       
-      def index_table_name(index)
-        "#{Dynamoid::Config.namespace}_index_#{self.to_s.downcase}_#{index.collect(&:to_s).collect(&:pluralize).join('_and_')}"
+      def find_index(index)
+        self.indexes[Array(index).collect(&:to_s).sort.collect(&:to_sym)]
       end
       
-      def key_for_index(index, values = [])
-        values = values.collect(&:to_s).sort.join('.')
+      def create_indexes
+        self.indexes.each do |name, index|
+          opts = index.range_key? ? {:range_key => :range} : {}
+          self.create_table(index.table_name, :id, opts) unless self.table_exists?(index.table_name)
+        end
       end
     end
     
-    def key_for_index(index)
-      self.class.key_for_index(index, index.collect{|i| self.send(i)})
-    end
-    
     def save_indexes
-      self.class.indexes.each do |index|
-        next if self.key_for_index(index).blank?
-        existing = Dynamoid::Adapter.read(self.class.index_table_name(index), self.key_for_index(index))
-        ids = existing ? existing[:ids] : Set.new
-        Dynamoid::Adapter.write(self.class.index_table_name(index), {:id => self.key_for_index(index), :ids => ids.merge([self.id])})
+      self.class.indexes.each do |name, index|
+        index.save(self)
       end
     end
     
     def delete_indexes
-      self.class.indexes.each do |index|
-        next if self.key_for_index(index).blank?
-        existing = Dynamoid::Adapter.read(self.class.index_table_name(index), self.key_for_index(index))
-        next unless existing && existing[:ids]
-        Dynamoid::Adapter.write(self.class.index_table_name(index), {:id => self.key_for_index(index), :ids => existing[:ids] - [self.id]})
+      self.class.indexes.each do |name, index|
+        index.delete(self)
       end
     end
   end
   
-end
+end