dyna_model 0.0.1

data/lib/dyna_model/extensions/symbol.rb

@@ -0,0 +1,11 @@
+class Symbol
+
+  DynaModel::Table::COMPARISON_OPERATOR.keys.each do |oper|
+    class_eval <<-OPERATORS
+      def #{oper}
+        "\#\{self.to_s\}.#{oper}"
+      end
+    OPERATORS
+  end
+
+end

data/lib/dyna_model/persistence.rb

@@ -0,0 +1,77 @@
+# TODO: optimistic locking?
+
+module DynaModel
+  module Persistence
+    extend ActiveSupport::Concern
+
+    private
+    def populate_id
+      #@_id = UUIDTools::UUID.random_create.to_s.downcase
+    end
+
+    private
+    def dynamo_db_table
+      self.class.dynamo_db_table(shard)
+    end
+
+    private
+    def create_storage
+      run_callbacks :save do
+        run_callbacks :create do
+          self.class.dynamo_db_table.write(serialize_attributes)
+        end
+      end
+    end
+
+    private
+    def update_storage
+      # Only enumerating dirty (i.e. changed) attributes.  Empty
+      # (nil and empty set) values are deleted, the others are replaced.
+      attr_updates = {}
+      changed.each do |attr_name|
+        attribute = self.class.attribute_for(attr_name)
+        value = serialize_attribute(attribute, @_data[attr_name])
+        if value.nil? or value == []
+          attr_updates[attr_name] = nil
+        else
+          attr_updates[attr_name] = value
+        end
+      end
+
+      run_callbacks :save do
+        run_callbacks :update do
+          self.class.dynamo_db_table.write(attr_updates, {
+            update_item: dynamo_db_item_key_values,
+            shard_name: self.shard
+          })
+        end
+      end
+    end
+
+    private
+    def delete_storage
+      run_callbacks :destroy do
+        self.class.dynamo_db_table.delete_item(
+          delete_item: dynamo_db_item_key_values,
+          shard_name: self.shard
+        )
+      end
+    end
+
+    private
+    def deserialize_item_data data
+      data.inject({}) do |hash,(attr_name,value)|
+        if attribute = self.class.attributes[attr_name]
+          hash[attr_name] = value.is_a?(Set) ?
+            value.map{|v| attribute.deserialize(v) } :
+            attribute.deserialize(value)
+        end
+        hash
+      end
+    end
+
+    module ClassMethods
+    end
+
+  end
+end

data/lib/dyna_model/query.rb

@@ -0,0 +1,207 @@
+module DynaModel
+  module Query
+    extend ActiveSupport::Concern
+
+    # Failsafe
+    QUERY_TIMEOUT = 30 # seconds
+    DEFAULT_BATCH_SIZE = 100
+
+    module ClassMethods
+
+      def read_guid(guid, options={})
+        return nil if guid.blank?
+        if self.range_key
+          hash_value, range_value = guid.split(self.guid_delimiter)
+          self.read(hash_value, range_value, options)
+        else
+          self.read(guid, options)
+        end
+      end
+
+      def read(hash_value, range_value_or_options=nil, options=nil)
+        if self.range_key.nil?
+          item_attrs = self.dynamo_db_table.get_item(hash_value, range_value_or_options || {})[:item]
+          return nil if item_attrs.nil?
+          self.obj_from_attrs(Table.values_from_response_hash(item_attrs), (range_value_or_options || {}))
+        else
+          raise ArgumentError, "This table requires a range_key_value" if range_value_or_options.nil?
+          self.read_range(hash_value, (options || {}).merge(range: { self.range_key[:attribute_name].to_sym.eq => range_value_or_options})).first
+        end
+      end
+
+      def read_multiple(keys, options={})
+        results_map = {}
+        results = self.dynamo_db_table.batch_get_item(keys, options)
+        results[:responses][self.dynamo_db_table_name(options[:shard_name])].each do |result|
+          attrs = Response.strip_attr_types(result)
+          obj = self.obj_from_attrs(attrs, options)
+          if self.dynamo_db_table.range_keys.present? && primary_range_key = self.dynamo_db_table.range_keys.find{|rk| rk[:primary_range_key] }
+            (results_map[attrs[self.dynamo_db_table.hash_key[:attribute_name]]] ||= {})[attrs[primary_range_key[:attribute_name]]] = obj
+          else
+            results_map[attrs[self.dynamo_db_table.hash_key[:attribute_name]]] = obj
+          end
+        end
+        results_map
+      end
+
+      # Read results up to the limit
+      #   read_range("1", :range => { :varname.gte => "2"}, :limit => 10)
+      # Loop results in given batch size until limit is hit or no more results
+      #   read_range("1", :range => { :varname.eq => "2"}, :batch => 10, :limit => 1000)
+      def read_range(hash_value, options={})
+        raise ArgumentError, "no range_key specified for this table" if self.dynamo_db_table.range_keys.blank? && self.global_secondary_indexes.blank?
+        aggregated_results = []
+
+        # Useful if doing pagination where you would need the last key evaluated
+        return_last_evaluated_key = options.delete(:return_last_evaluated_key)
+        batch_size = options.delete(:batch) || DEFAULT_BATCH_SIZE
+        max_results_limit = options[:limit]
+        if options[:limit] && options[:limit] > batch_size
+          options.merge!(:limit => batch_size)
+        end
+
+        results = self.dynamo_db_table.query(hash_value, options)
+        response = Response.new(results)
+
+        results[:member].each do |result|
+          attrs = Response.strip_attr_types(result)
+          aggregated_results << self.obj_from_attrs(attrs, options)
+        end
+
+        if response.more_results?
+          results_returned = response.count
+          batch_iteration = 0
+          Timeout::timeout(QUERY_TIMEOUT) do
+            while response.more_results?
+              if max_results_limit && (delta_results_limit = (max_results_limit-results_returned)) < batch_size
+                break if delta_results_limit == 0
+                options.merge!(limit: delta_results_limit)
+              else
+                options.merge!(limit: batch_size)
+              end
+
+              results = self.dynamo_db_table.query(hash_value, options.merge(exclusive_start_key: response.last_evaluated_key))
+              response = Response.new(results)
+              results[:member].each do |result|
+                attrs = Response.strip_attr_types(result)
+                aggregated_results << self.obj_from_attrs(attrs, options)
+              end
+              results_returned += response.count
+              batch_iteration += 1
+            end
+          end
+        end
+
+        if return_last_evaluated_key
+          {
+            last_evaluated_key: response.last_evaluated_key,
+            members: aggregated_results
+          }
+        else
+          aggregated_results
+        end
+      end
+
+      def count_range(hash_value, options={})
+        raise ArgumentError, "no range_key specified for this table" if self.dynamo_db_table.range_keys.blank?
+        results = self.dynamo_db_table.query(hash_value, options.merge(select: :count))
+        Response.new(results).count
+      end
+
+      def read_first(hash_value, options={})
+        self.read_range(hash_value, options).first
+      end
+
+      #:count=>10, :scanned_count=>10, :last_evaluated_key=>{"guid"=>{:s=>"11f82550-5c5d-11e3-9b55-d311a43114ca"}}}
+      # :manual_batching => true|false
+      #   return results with last_evaluated_key instead of automatically looping through (useful to throttle or )
+      def scan(options={})
+        aggregated_results = []
+
+        batch_size = options.delete(:batch) || DEFAULT_BATCH_SIZE
+        max_results_limit = options[:limit]
+        options[:limit] = batch_size
+
+        results = self.dynamo_db_table.scan(options)
+        response = Response.new(results)
+
+        results[:member].each do |result|
+          attrs = Response.strip_attr_types(result)
+          aggregated_results << self.obj_from_attrs(attrs, options)
+        end
+
+        if response.more_results? && !options[:manual_batching]
+          results_returned = response.count
+          batch_iteration = 0
+          Timeout::timeout(QUERY_TIMEOUT) do
+            while response.more_results?
+              if max_results_limit && (delta_results_limit = (max_results_limit-results_returned)) < batch_size
+                break if delta_results_limit == 0
+                options.merge!(limit: delta_results_limit)
+              else
+                options.merge!(limit: batch_size)
+              end
+
+              results = dynamo_table.scan(options.merge(exclusive_start_key: response.last_evaluated_key))
+              response = Response.new(results)
+              results[:member].each do |result|
+                attrs = Response.strip_attr_types(result)
+                aggregated_results << self.obj_from_attrs(attrs, options)
+              end
+              results_returned += response.count
+              batch_iteration += 1
+            end
+          end
+        end
+
+        if options[:manual_batching]
+          response_hash = {
+            results: aggregated_results,
+            last_evaluated_key: results[:last_evaluated_key]
+          }
+          response_hash.merge!(consumed_capacity: results[:consumed_capacity]) if results[:consumed_capacity]
+          response_hash
+        else
+          aggregated_results
+        end
+      end # scan
+
+      protected
+      def obj_from_attrs(attrs, options={})
+        obj = self.new(shard: self.shard_name(options[:shard_name]))
+        obj.send(:hydrate, nil, attrs)
+        if options[:select]
+          obj.instance_variable_set("@_select", options[:select])
+          if options[:select] != :all
+            #:all, :projected, :count, :specific
+            selected_attrs = []
+            # Primary hash/range key are always returned...
+            self.table_schema[:key_schema].each do |k|
+              selected_attrs << k[:attribute_name]
+            end
+            if options[:select] == :projected
+              index = ((self.table_schema[:global_secondary_indexes] || []) + (self.table_schema[:local_secondary_indexes] || [])).find { |i| i[:index_name] == options[:index_name].to_s }
+              raise "Index '#{options[:index_name]}' not found in table schema" unless index
+              index[:key_schema].each do |k|
+                selected_attrs << k[:attribute_name].to_s
+              end
+              if index[:projection] && index[:projection][:non_key_attributes]
+                index[:projection][:non_key_attributes].each do |a|
+                  selected_attrs << a.to_s
+                end
+              end
+            elsif options[:select].is_a?(Array)
+              obj.instance_variable_set("@_select", :specific)
+              selected_attrs += options[:select].map(&:to_s)
+            end
+            selected_attrs.uniq!
+            obj.instance_variable_set("@_selected_attributes", selected_attrs.compact)
+          end
+        end
+        obj
+      end
+
+    end # ClassMethods
+
+  end
+end

data/lib/dyna_model/response.rb

@@ -0,0 +1,36 @@
+module DynaModel
+  class Response
+
+    def initialize(response)
+      raise ArgumentError, "response should be an AWS::Core::Response" unless response.is_a?(AWS::Core::Response)
+      @raw_response = response
+    end
+
+    #def values_from_response_hash(options = {})
+      #@raw_response.inject({}) do |h, (key, value_hash)|
+        #h.update(key => value_hash.to_a.last)
+      #end
+    #end
+
+    def count
+      @raw_response[:count]
+    end
+
+    def last_evaluated_key
+      @raw_response[:last_evaluated_key]
+    end
+
+    def more_results?
+      @raw_response.has_key?(:last_evaluated_key)
+    end
+
+    def self.strip_attr_types(hash)
+      attrs = {}
+      hash.each_pair do |k,v|
+        attrs[k] = v.values.first
+      end
+      attrs
+    end
+
+  end
+end

data/lib/dyna_model/schema.rb

@@ -0,0 +1,251 @@
+module DynaModel
+  module Schema
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+
+      KEY_TYPE = {
+        hash: "HASH",
+        range: "RANGE"
+      }
+
+      PROJECTION_TYPE = {
+        keys_only: "KEYS_ONLY",
+        all: "ALL",
+        include: "INCLUDE"
+      }
+
+      ATTR_TYPES = {
+        AWS::Record::Attributes::StringAttr => "S",
+        AWS::Record::Attributes::IntegerAttr => "N",
+        AWS::Record::Attributes::FloatAttr => "N",
+        AWS::Record::Attributes::BooleanAttr => "S",
+        AWS::Record::Attributes::DateTimeAttr => "N",
+        AWS::Record::Attributes::DateAttr => "N",
+        AWS::Record::Attributes::SerializedAttr => "B"
+      }
+
+      def table_schema
+        schema = {
+          table_name: dynamo_db_table_name,
+          provisioned_throughput: {
+            read_capacity_units: read_provision,
+            write_capacity_units: write_provision
+          },
+          key_schema: key_schema,
+          attribute_definitions: attribute_definitions
+        }
+        schema[:local_secondary_indexes] = local_secondary_indexes unless local_secondary_indexes.blank?
+        schema[:global_secondary_indexes] = global_secondary_indexes unless global_secondary_indexes.blank?
+        schema
+      end
+
+      def guid_delimiter(val=nil)
+        if val
+          raise(ArgumentError, "Invalid guid_delimiter") if val.blank?
+          @guid_delimiter = val.to_s
+        else
+          @guid_delimiter || DynaModel::Config.default_guid_delimiter
+        end
+      end
+
+      def read_provision(val=nil)
+        if val
+          raise(ArgumentError, "Invalid read provision") unless val.to_i >= 1
+          @dynamo_read_provision = val.to_i
+        else
+          @dynamo_read_provision || DynaModel::Config.read_provision
+        end
+      end
+
+      def write_provision(val=nil)
+        if val
+          raise(ArgumentError, "Invalid write provision") unless val.to_i >= 1
+          @dynamo_write_provision = val.to_i
+        else
+          @dynamo_write_provision || DynaModel::Config.write_provision
+        end
+      end
+
+      def hash_key(hash_key_key=nil)
+        if hash_key_key
+          hash_key_attribute = self.attributes[hash_key_key.to_s]
+          raise(ArgumentError, "Could not find attribute definition for hash_key #{hash_key_key}") unless hash_key_attribute
+          raise(ArgumentError, "Invalid attribute type for hash_key") unless [AWS::Record::Attributes::StringAttr, AWS::Record::Attributes::IntegerAttr, AWS::Record::Attributes::FloatAttr].include?(hash_key_attribute.class)
+
+          validates_presence_of hash_key_attribute.name.to_sym
+
+          @dynamo_hash_key = {
+            attribute_name: hash_key_attribute.name,
+            key_type: KEY_TYPE[:hash]
+          }
+        else
+          @dynamo_hash_key
+        end
+      end
+
+      def range_key(range_key_key=nil)
+        if range_key_key
+          range_key_attribute = self.attributes[range_key_key.to_s]
+          raise(ArgumentError, "Could not find attribute definition for range_key #{range_key_key}") unless range_key_attribute
+          raise(ArgumentError, "Invalid attribute type for range_key") unless [AWS::Record::Attributes::StringAttr, AWS::Record::Attributes::IntegerAttr, AWS::Record::Attributes::FloatAttr].include?(range_key_attribute.class)
+
+          validates_presence_of range_key_attribute.name.to_sym
+
+          @dynamo_range_key = {
+            attribute_name: range_key_attribute.name,
+            key_type: KEY_TYPE[:range]
+          }
+        else
+          @dynamo_range_key
+        end
+      end
+
+      # TODO - need to add projections?
+      def attribute_definitions
+        # Keys for hash/range/secondary
+        # S | N | B
+
+        keys = []
+        keys << hash_key[:attribute_name]
+        keys << range_key[:attribute_name] if range_key
+        local_secondary_indexes.each do |lsi|
+          keys << lsi[:key_schema].select{|h| h[:key_type] == "RANGE"}.first[:attribute_name]
+        end
+
+        global_secondary_indexes.each do |lsi|
+          lsi[:key_schema].each do |a|
+            keys << a[:attribute_name]
+          end
+        end
+
+        definitions = keys.uniq.collect do |k|
+          attr = self.attributes[k.to_s]
+          {
+            attribute_name: attr.name,
+            attribute_type: attribute_type_indicator(attr)
+          }
+        end
+      end
+
+      def attribute_type_indicator(attr)
+        if attr_type = ATTR_TYPES[attr.class]
+          attr_type
+        else
+          raise "unsupported attribute type #{attr.class}"
+        end
+      end
+
+      def key_schema
+        raise(ArgumentError, 'hash_key was not set for this table') if @dynamo_hash_key.blank?
+        schema = [hash_key]
+        schema << range_key if range_key 
+        schema
+      end
+
+      def global_secondary_indexes
+        @global_secondary_indexes ||= []
+      end
+
+      # { hash_key: :hash_key_here, range_key: :optional_range_key_here }
+      # :name
+      # :projection
+      # :read_provision
+      # :write_provision
+      def global_secondary_index(index_name, options={})
+        options[:projection] ||= :keys_only
+        global_secondary_index_hash = {
+          projection: {},
+          provisioned_throughput: {
+            read_capacity_units: options[:read_provision] || read_provision,
+            write_capacity_units: options[:write_provision] || write_provision
+          }
+        }
+        if options[:projection].is_a?(Array) && options[:projection].size > 0
+          options[:projection].each do |non_key_attr|
+            attr = self.attributes[non_key_attr.to_s]
+            raise(ArgumentError, "Could not find attribute definition for projection on #{non_key_attr}") unless attr
+            (global_secondary_index_hash[:projection][:non_key_attributes] ||= []) << attr.name
+          end
+          global_secondary_index_hash[:projection][:projection_type] = PROJECTION_TYPE[:include]
+        else
+          raise(ArgumentError, 'projection must be :all, :keys_only, Array (or attrs)') unless options[:projection] == :keys_only || options[:projection] == :all
+          global_secondary_index_hash[:projection][:projection_type] = PROJECTION_TYPE[options[:projection]]
+        end
+
+        if !options.has_key?(:hash_key) || self.attributes[options[:hash_key].to_s].blank?
+          raise(ArgumentError, "Could not find attribute definition for global secondary index on hash_key specified")
+        end
+        hash_key_attr = self.attributes[options[:hash_key].to_s]
+
+        if options.has_key?(:range_key) && self.attributes[options[:range_key].to_s].blank?
+          raise(ArgumentError, "Could not find attribute definition for global secondary index on range_key specified")
+        end
+        range_key_attr = nil
+        range_key_attr = self.attributes[options[:range_key].to_s] if options.has_key?(:range_key)
+
+        ## Force naming of index_name for lookup later
+        #global_secondary_index_hash[:index_name] = (index_name.to_s || "#{hash_key_attr.name}#{"_#{range_key_attr.name}" if range_key_attr}_gsi_index".camelcase)
+        global_secondary_index_hash[:index_name] = index_name.to_s
+
+        global_secondary_index_hash[:key_schema] = [
+          {
+            attribute_name: hash_key_attr.name,
+            key_type: KEY_TYPE[:hash]
+          }
+        ]
+        global_secondary_index_hash[:key_schema] << {
+          attribute_name: range_key_attr.name,
+          key_type: KEY_TYPE[:range]
+        } if range_key_attr
+
+        return false if (@global_secondary_indexes ||= []).select {|i| i[:index_name] == global_secondary_index_hash[:index_name] }.present? # Do not add if we already have a range key set for this attr
+        (@global_secondary_indexes ||= []) << global_secondary_index_hash
+      end
+
+      def local_secondary_indexes
+        @local_secondary_indexes ||= []
+      end
+
+      def local_secondary_index(range_key_attr, options={})
+        options[:projection] ||= :keys_only
+        local_secondary_index_hash = {
+          projection: {}
+        }
+        if options[:projection].is_a?(Array) && options[:projection].size > 0
+          options[:projection].each do |non_key_attr|
+            attr = self.attributes[non_key_attr.to_s]
+            raise(ArgumentError, "Could not find attribute definition for projection on #{non_key_attr}") unless attr
+            (local_secondary_index_hash[:projection][:non_key_attributes] ||= []) << attr.name
+          end
+          local_secondary_index_hash[:projection][:projection_type] = PROJECTION_TYPE[:include]
+        else
+          raise(ArgumentError, 'projection must be :all, :keys_only, Array (or attrs)') unless options[:projection] == :keys_only || options[:projection] == :all
+          local_secondary_index_hash[:projection][:projection_type] = PROJECTION_TYPE[options[:projection]]
+        end
+
+        range_attr = self.attributes[range_key_attr.to_s]
+        raise(ArgumentError, "Could not find attribute definition for local secondary index on #{range_key_attr}") unless range_attr
+        local_secondary_index_hash[:index_name] = (options[:name] || options[:index_name] || "#{range_attr.name}_index".camelcase)
+
+        hash_key_attr = self.attributes[hash_key[:attribute_name].to_s]
+        raise(ArgumentError, "Could not find attribute definition for hash_key") unless hash_key_attr
+
+        local_secondary_index_hash[:key_schema] = [
+          {
+            attribute_name: hash_key_attr.name,
+            key_type: KEY_TYPE[:hash]
+          },
+          {
+            attribute_name: range_attr.name,
+            key_type: KEY_TYPE[:range]
+          }
+        ]
+        return false if (@local_secondary_indexes ||= []).select {|i| i[:index_name] == local_secondary_index_hash[:index_name] }.present? # Do not add if we already have a range key set for this attr
+        (@local_secondary_indexes ||= []) << local_secondary_index_hash
+      end
+
+    end # ClassMethods
+
+  end
+end