dynamoid-moda 0.7.1

data/lib/dynamoid.rb

@@ -0,0 +1,46 @@
+require "delegate"
+require "time"
+require "securerandom"
+require "active_support/core_ext"
+require 'active_support/json'
+require "active_support/inflector"
+require "active_support/lazy_load_hooks"
+require "active_support/time_with_zone"
+require "active_model"
+
+require 'dynamoid/errors'
+require 'dynamoid/fields'
+require 'dynamoid/indexes'
+require 'dynamoid/associations'
+require 'dynamoid/persistence'
+require 'dynamoid/dirty'
+require 'dynamoid/validations'
+require 'dynamoid/criteria'
+require 'dynamoid/finders'
+require 'dynamoid/identity_map'
+require 'dynamoid/config'
+require 'dynamoid/components'
+require 'dynamoid/document'
+require 'dynamoid/adapter'
+
+require 'dynamoid/middleware/identity_map'
+
+module Dynamoid
+  extend self
+
+  MAX_ITEM_SIZE = 65_536
+  
+  def configure
+    block_given? ? yield(Dynamoid::Config) : Dynamoid::Config
+  end
+  alias :config :configure
+  
+  def logger
+    Dynamoid::Config.logger
+  end
+  
+  def included_models
+    @included_models ||= []
+  end
+  
+end

data/lib/dynamoid/adapter.rb

@@ -0,0 +1,267 @@
+# encoding: utf-8
+module Dynamoid
+
+  # Adapter provides a generic, write-through class that abstracts variations in the underlying connections to provide a uniform response
+  # to Dynamoid.
+  module Adapter
+    extend self
+    attr_accessor :tables
+
+    # The actual adapter currently in use: presently AwsSdk.
+    #
+    # @since 0.2.0
+    def adapter
+      reconnect! unless @adapter
+      @adapter
+    end
+
+    # Establishes a connection to the underyling adapter and caches all its tables for speedier future lookups. Issued when the adapter is first called.
+    #
+    # @since 0.2.0
+    def reconnect!
+      require "dynamoid/adapter/#{Dynamoid::Config.adapter}" unless Dynamoid::Adapter.const_defined?(Dynamoid::Config.adapter.camelcase)
+      @adapter = Dynamoid::Adapter.const_get(Dynamoid::Config.adapter.camelcase)
+      @adapter.connect! if @adapter.respond_to?(:connect!)
+      self.tables = benchmark('Cache Tables') {list_tables}
+    end
+
+    # Shows how long it takes a method to run on the adapter. Useful for generating logged output.
+    #
+    # @param [Symbol] method the name of the method to appear in the log
+    # @param [Array] args the arguments to the method to appear in the log
+    # @yield the actual code to benchmark
+    #
+    # @return the result of the yield
+    #
+    # @since 0.2.0
+    def benchmark(method, *args)
+      start = Time.now
+      result = yield
+      Dynamoid.logger.info "(#{((Time.now - start) * 1000.0).round(2)} ms) #{method.to_s.split('_').collect(&:upcase).join(' ')}#{ " - #{args.inspect}" unless args.nil? || args.empty? }"
+      return result
+    end
+
+    # Write an object to the adapter. Partition it to a randomly selected key first if necessary.
+    #
+    # @param [String] table the name of the table to write the object to
+    # @param [Object] object the object itself
+    # @param [Hash] options Options that are passed to the put_item call
+    #
+    # @return [Object] the persisted object
+    #
+    # @since 0.2.0
+    def write(table, object, options = nil)
+      if Dynamoid::Config.partitioning? && object[:id]
+        object[:id] = "#{object[:id]}.#{Random.rand(Dynamoid::Config.partition_size)}"
+        object[:updated_at] = Time.now.to_f
+      end
+      put_item(table, object, options)
+    end
+
+    # Read one or many keys from the selected table. This method intelligently calls batch_get or get on the underlying adapter depending on
+    # whether ids is a range or a single key: additionally, if partitioning is enabled, it batch_gets all keys in the partition space
+    # automatically. Finally, if a range key is present, it will also interpolate that into the ids so that the batch get will acquire the
+    # correct record.
+    #
+    # @param [String] table the name of the table to write the object to
+    # @param [Array] ids to fetch, can also be a string of just one id
+    # @param [Hash] options: Passed to the underlying query. The :range_key option is required whenever the table has a range key,
+    #                        unless multiple ids are passed in and Dynamoid::Config.partitioning? is turned off.
+    #
+    # @since 0.2.0
+    def read(table, ids, options = {})
+      range_key = options.delete(:range_key)
+
+      if ids.respond_to?(:each)
+        ids = ids.collect{|id| range_key ? [id, range_key] : id}
+        if Dynamoid::Config.partitioning?
+          results = batch_get_item({table => id_with_partitions(ids)}, options)
+          {table => result_for_partition(results[table],table)}
+        else
+          batch_get_item({table => ids}, options)
+        end
+      else
+        if Dynamoid::Config.partitioning?
+          ids = range_key ? [[ids, range_key]] : ids
+          results = batch_get_item({table => id_with_partitions(ids)}, options)
+          result_for_partition(results[table],table).first
+        else
+          options[:range_key] = range_key if range_key
+          get_item(table, ids, options)
+        end
+      end
+    end
+
+    # Delete an item from a table. If partitioning is turned on, deletes all partitioned keys as well.
+    #
+    # @param [String] table the name of the table to write the object to
+    # @param [Array] ids to delete, can also be a string of just one id
+    # @param [Array] range_key of the record to delete, can also be a string of just one range_key
+    #
+    def delete(table, ids, options = {})
+      range_key = options[:range_key] #array of range keys that matches the ids passed in
+      if ids.respond_to?(:each)
+        if range_key.respond_to?(:each)
+          #turn ids into array of arrays each element being hash_key, range_key
+          ids = ids.each_with_index.map{|id,i| [id,range_key[i]]}
+        else
+          ids = range_key ? [[ids, range_key]] : ids
+        end
+        
+        if Dynamoid::Config.partitioning?
+          batch_delete_item(table => id_with_partitions(ids))
+        else
+          batch_delete_item(table => ids)
+        end
+      else
+        if Dynamoid::Config.partitioning?
+          ids = range_key ? [[ids, range_key]] : ids
+          batch_delete_item(table => id_with_partitions(ids))
+        else
+          delete_item(table, ids, options)
+        end
+      end
+    end
+
+    # Scans a table. Generally quite slow; try to avoid using scan if at all possible.
+    #
+    # @param [String] table the name of the table to write the object to
+    # @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan
+    #
+    # @since 0.2.0
+    def scan(table, query, opts = {})
+      if Dynamoid::Config.partitioning?
+        results = benchmark('Scan', table, query) {adapter.scan(table, query, opts)}
+        result_for_partition(results,table)
+      else
+        benchmark('Scan', table, query) {adapter.scan(table, query, opts)}
+      end
+    end
+
+    [:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item].each do |m|
+      # Method delegation with benchmark to the underlying adapter. Faster than relying on method_missing.
+      #
+      # @since 0.2.0
+      define_method(m) do |*args|
+        benchmark("#{m.to_s}", args) {adapter.send(m, *args)}
+      end
+    end
+
+    # Takes a list of ids and returns them with partitioning added. If an array of arrays is passed, we assume the second key is the range key
+    # and pass it in unchanged.
+    #
+    # @example Partition id 1
+    #   Dynamoid::Adapter.id_with_partitions(['1']) # ['1.0', '1.1', '1.2', ..., '1.199']
+    # @example Partition id 1 and range_key 1.0
+    #   Dynamoid::Adapter.id_with_partitions([['1', 1.0]]) # [['1.0', 1.0], ['1.1', 1.0], ['1.2', 1.0], ..., ['1.199', 1.0]]
+    #
+    # @param [Array] ids array of ids to partition
+    #
+    # @since 0.2.0
+    def id_with_partitions(ids)
+      Array(ids).collect {|id| (0...Dynamoid::Config.partition_size).collect{|n| id.is_a?(Array) ? ["#{id.first}.#{n}", id.last] : "#{id}.#{n}"}}.flatten(1)
+    end
+    
+    #Get original id (hash_key) and partiton number from a hash_key
+    #
+    # @param [String] id the id or hash_key of a record, ex. xxxxx.13
+    #
+    # @return [String,String] original_id and the partition number, ex original_id = xxxxx partition = 13
+    def get_original_id_and_partition id
+      partition = id.split('.').last
+      id = id.split(".#{partition}").first
+
+      return id, partition
+    end
+
+    # Takes an array of query results that are partitioned, find the most recently updated ones that share an id and range_key, and return only the most recently updated. Compares each result by
+    # their id and updated_at attributes; if the updated_at is the greatest, then it must be the correct result.
+    #
+    # @param [Array] returned partitioned results from a query
+    # @param [String] table_name the name of the table
+    #
+    # @since 0.2.0
+    def result_for_partition(results, table_name)
+      table = adapter.get_table(table_name)
+      
+      if table.range_key     
+        range_key_name = table.range_key.name.to_sym
+        
+        final_hash = {}
+
+        results.each do |record|
+          test_record = final_hash[record[range_key_name]]
+          
+          if test_record.nil? || ((record[range_key_name] == test_record[range_key_name]) && (record[:updated_at] > test_record[:updated_at]))
+            #get ride of our partition and put it in the array with the range key
+            record[:id], partition = get_original_id_and_partition  record[:id]
+            final_hash[record[range_key_name]] = record
+          end
+        end
+  
+        return final_hash.values
+      else
+        {}.tap do |hash|
+          Array(results).each do |result|
+            next if result.nil?
+            #Need to find the value of id with out the . and partition number
+            id, partition = get_original_id_and_partition result[:id]
+  
+            if !hash[id] || (result[:updated_at] > hash[id][:updated_at])
+              result[:id] = id
+              hash[id] = result
+            end
+          end
+        end.values
+      end
+    end
+
+    # Delegate all methods that aren't defind here to the underlying adapter.
+    #
+    # @since 0.2.0
+    def method_missing(method, *args, &block)
+      return benchmark(method, *args) {adapter.send(method, *args, &block)} if @adapter.respond_to?(method)
+      super
+    end
+    
+    # Query the DynamoDB table. This employs DynamoDB's indexes so is generally faster than scanning, but is
+    # only really useful for range queries, since it can only find by one hash key at once. Only provide
+    # one range key to the hash. If paritioning is on, will run a query for every parition and join the results
+    #
+    # @param [String] table_name the name of the table
+    # @param [Hash] opts the options to query the table with
+    # @option opts [String] :hash_value the value of the hash key to find
+    # @option opts [Range] :range_value find the range key within this range
+    # @option opts [Number] :range_greater_than find range keys greater than this
+    # @option opts [Number] :range_less_than find range keys less than this
+    # @option opts [Number] :range_gte find range keys greater than or equal to this
+    # @option opts [Number] :range_lte find range keys less than or equal to this
+    #
+    # @return [Array] an array of all matching items
+    #
+    def query(table_name, opts = {})
+      
+      unless Dynamoid::Config.partitioning?
+        #no paritioning? just pass to the standard query method
+        adapter.query(table_name, opts)
+      else
+        #get all the hash_values that could be possible
+        ids = id_with_partitions(opts[:hash_value])
+
+        #lets not overwrite with the original options
+        modified_options = opts.clone     
+        results = []
+        
+        #loop and query on each of the partition ids
+        ids.each do |id|
+          modified_options[:hash_value] = id
+
+          query_result = adapter.query(table_name, modified_options)
+          results += query_result.inject([]){|array, result| array += [result]} if query_result.any?
+        end 
+
+        result_for_partition results, table_name
+      end
+    end
+  end
+end

data/lib/dynamoid/adapter/aws_sdk.rb

@@ -0,0 +1,309 @@
+# encoding: utf-8
+require 'aws'
+
+module Dynamoid
+  module Adapter
+
+    # The AwsSdk adapter provides support for the AWS-SDK for Ruby gem.
+    # More information is available at that Gem's Github page:
+    # https://github.com/amazonwebservices/aws-sdk-for-ruby
+    #
+    module AwsSdk
+      extend self
+      @@connection = nil
+
+      # Establish the connection to DynamoDB.
+      #
+      # @return [AWS::DynamoDB::Connection] the raw DynamoDB connection
+      # Call DynamoDB new, with no parameters. 
+      # Make sure the aws.yml file or aws.rb file, refer the link for more details. 
+      #https://github.com/amazonwebservices/aws-sdk-for-ruby
+      # 1. Create config/aws.yml as follows:
+      # Fill in your AWS Access Key ID and Secret Access Key
+      # http://aws.amazon.com/security-credentials
+      #access_key_id: REPLACE_WITH_ACCESS_KEY_ID
+      #secret_access_key: REPLACE_WITH_SECRET_ACCESS_KEY
+      #(or)
+      #2, Create config/initializers/aws.rb as follows:
+      # load the libraries
+      #require 'aws'
+      # log requests using the default rails logger
+      #AWS.config(:logger => Rails.logger)
+      # load credentials from a file
+      #config_path = File.expand_path(File.dirname(__FILE__)+"/../aws.yml")
+      #AWS.config(YAML.load(File.read(config_path)))
+      #Additionally include any of the dynamodb paramters as needed 
+      #(eg: if you would like to change the dynamodb endpoint, then add the parameter in 
+      # the following paramter in the file  aws.yml or aws.rb 
+      # dynamo_db_endpoint : dynamodb.ap-southeast-1.amazonaws.com)
+      # @since 0.2.0
+      def connect!
+        @@connection = AWS::DynamoDB.new
+      end
+
+      # Return the established connection.
+      #
+      # @return [AWS::DynamoDB::Connection] the raw DynamoDB connection
+      #
+      # @since 0.2.0
+      def connection
+        @@connection
+      end
+
+      # Get many items at once from DynamoDB. More efficient than getting each item individually.
+      #
+      # @example Retrieve IDs 1 and 2 from the table testtable
+      #   Dynamoid::Adapter::AwsSdk.batch_get_item({'table1' => ['1', '2']}, :consistent_read => true)
+      #
+      # @param [Hash] table_ids the hash of tables and IDs to retrieve
+      # @param [Hash] options to be passed to underlying BatchGet call
+      #
+      # @return [Hash] a hash where keys are the table names and the values are the retrieved items
+      #
+      # @since 0.2.0
+      def batch_get_item(table_ids, options = {})
+        hash = Hash.new{|h, k| h[k] = []}
+        return hash if table_ids.all?{|k, v| v.empty?}
+        table_ids.each do |t, ids|
+          Array(ids).in_groups_of(100, false) do |group|
+            batch = AWS::DynamoDB::BatchGet.new(:config => @@connection.config)
+            batch.table(t, :all, Array(group), options) unless group.nil? || group.empty?
+            batch.each do |table_name, attributes|
+              hash[table_name] << attributes.symbolize_keys!
+            end
+          end
+        end
+        hash
+      end
+
+      # Delete many items at once from DynamoDB. More efficient than delete each item individually.
+      #
+      # @example Delete IDs 1 and 2 from the table testtable
+      #   Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => ['1', '2'])
+      #or
+      #   Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => [['hk1', 'rk2'], ['hk1', 'rk2']]]))
+      #
+      # @param [Hash] options the hash of tables and IDs to delete
+      #
+      # @return nil
+      #
+      def batch_delete_item(options)
+        return nil if options.all?{|k, v| v.empty?}
+        options.each do |t, ids|
+          Array(ids).in_groups_of(25, false) do |group|
+            batch = AWS::DynamoDB::BatchWrite.new(:config => @@connection.config)
+            batch.delete(t,group)
+            batch.process!
+          end
+        end
+        nil
+      end
+
+      # Create a table on DynamoDB. This usually takes a long time to complete.
+      #
+      # @param [String] table_name the name of the table to create
+      # @param [Symbol] key the table's primary key (defaults to :id)
+      # @param [Hash] options provide a range_key here if you want one for the table
+      #
+      # @since 0.2.0
+      def create_table(table_name, key = :id, options = {})
+        Dynamoid.logger.info "Creating #{table_name} table. This could take a while."
+        options[:hash_key] ||= {key.to_sym => :string}
+        read_capacity = options[:read_capacity] || Dynamoid::Config.read_capacity
+        write_capacity = options[:write_capacity] || Dynamoid::Config.write_capacity
+        table = @@connection.tables.create(table_name, read_capacity, write_capacity, options)
+        sleep 0.5 while table.status == :creating
+        return table
+      end
+
+      # Removes an item from DynamoDB.
+      #
+      # @param [String] table_name the name of the table
+      # @param [String] key the hash key of the item to delete
+      # @param [Number] range_key the range key of the item to delete, required if the table has a composite key
+      #
+      # @since 0.2.0
+      def delete_item(table_name, key, options = {})
+        range_key = options.delete(:range_key)
+        table = get_table(table_name)
+        result = table.items.at(key, range_key)
+        result.delete unless result.attributes.to_h.empty?
+        true
+      end
+
+      # Deletes an entire table from DynamoDB. Only 10 tables can be in the deleting state at once,
+      # so if you have more this method may raise an exception.
+      #
+      # @param [String] table_name the name of the table to destroy
+      #
+      # @since 0.2.0
+      def delete_table(table_name)
+        Dynamoid.logger.info "Deleting #{table_name} table. This could take a while."
+        table = @@connection.tables[table_name]
+        table.delete
+        sleep 0.5 while table.exists? == true
+      end
+
+      # @todo Add a DescribeTable method.
+
+      # Fetches an item from DynamoDB.
+      #
+      # @param [String] table_name the name of the table
+      # @param [String] key the hash key of the item to find
+      # @param [Number] range_key the range key of the item to find, required if the table has a composite key
+      #
+      # @return [Hash] a hash representing the raw item in DynamoDB
+      #
+      # @since 0.2.0
+      def get_item(table_name, key, options = {})
+        range_key = options.delete(:range_key)
+        table = get_table(table_name)
+
+        result = table.items.at(key, range_key).attributes.to_h(options)
+
+        if result.empty?
+          nil
+        else
+          result.symbolize_keys!
+        end
+      end
+
+      def update_item(table_name, key, options = {}, &block)
+        range_key = options.delete(:range_key)
+        conditions = options.delete(:conditions) || {}
+        table = get_table(table_name)
+        item = table.items.at(key, range_key)
+        item.attributes.update(conditions.merge(:return => :all_new), &block)
+      rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException
+        raise Dynamoid::Errors::ConditionalCheckFailedException
+      end
+
+      # List all tables on DynamoDB.
+      #
+      # @since 0.2.0
+      def list_tables
+        @@connection.tables.collect(&:name)
+      end
+
+      # Persists an item on DynamoDB.
+      #
+      # @param [String] table_name the name of the table
+      # @param [Object] object a hash or Dynamoid object to persist
+      #
+      # @since 0.2.0
+      def put_item(table_name, object, options = nil)
+        table = get_table(table_name)
+        table.items.create(
+            object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)},
+            options || {}
+        )
+      rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
+        raise Dynamoid::Errors::ConditionalCheckFailedException
+      end
+
+      # Query the DynamoDB table. This employs DynamoDB's indexes so is generally faster than scanning, but is
+      # only really useful for range queries, since it can only find by one hash key at once. Only provide
+      # one range key to the hash.
+      #
+      # @param [String] table_name the name of the table
+      # @param [Hash] opts the options to query the table with
+      # @option opts [String] :hash_value the value of the hash key to find
+      # @option opts [Range] :range_value find the range key within this range
+      # @option opts [Number] :range_greater_than find range keys greater than this
+      # @option opts [Number] :range_less_than find range keys less than this
+      # @option opts [Number] :range_gte find range keys greater than or equal to this
+      # @option opts [Number] :range_lte find range keys less than or equal to this
+      #
+      # @return [Enumerator] an iterator of all matching items
+      #
+      # @since 0.2.0
+      def query(table_name, opts = {})
+        table = get_table(table_name)
+
+        Enumerator.new do |yielder|
+          consistent_opts = { :consistent_read => opts[:consistent_read] || false }
+          if table.composite_key?
+            table.items.query(opts).each do |data|
+              if opts.has_key? :select
+                yielder.yield data.attributes.symbolize_keys!
+              else
+                yielder.yield data.attributes.to_h(consistent_opts).symbolize_keys!
+              end
+            end
+          else
+            yielder.yield get_item(table_name, opts[:hash_value])
+          end
+        end
+      end
+
+      # Scan the DynamoDB table. This is usually a very slow operation as it naively filters all data on
+      # the DynamoDB servers.
+      #
+      # @param [String] table_name the name of the table
+      # @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan
+      #
+      # @return [Enumerator] an iterator of all matching items
+      #
+      # @since 0.2.0
+      def scan(table_name, scan_hash, select_opts)
+        table = get_table(table_name)
+        Enumerator.new do |yielder|
+          # MF: Add support for advanced scan filters
+          key = scan_hash.keys[0] if scan_hash && scan_hash.keys
+          exp = scan_hash[:expression] if scan_hash.has_key? :expression
+          case exp
+            when "IN"
+              table.items.where(key).in(*scan_hash[key]).select(select_opts).each do |data|
+                yielder.yield data.attributes.symbolize_keys!
+              end
+            when "BETWEEN"
+              table.items.where(key).between(scan_hash[key][0], scan_hash[key][1]).select(select_opts).each do |data|
+                yielder.yield data.attributes.symbolize_keys!
+              end
+            when "BEGINS_WITH"
+              table.items.where(key).between(scan_hash[key][0], scan_hash[key][1]).select(select_opts).each do |data|
+                yielder.yield data.attributes.symbolize_keys!
+              end
+            when "CONTAINS"
+              table.items.where(key).contains(scan_hash[key]).select(select_opts).each do |data|
+                yielder.yield data.attributes.symbolize_keys!
+              end
+            else
+              table.items.where(scan_hash).select(select_opts).each do |data|
+                yielder.yield data.attributes.symbolize_keys!
+              end
+          end
+        end
+      end
+
+      # @todo Add an UpdateItem method.
+
+      # @todo Add an UpdateTable method.
+
+      def get_table(table_name)
+        unless table = table_cache[table_name]
+          table = @@connection.tables[table_name]
+          table.load_schema
+          table_cache[table_name] = table
+        end
+        table
+      end
+
+      def table_cache
+        @table_cache ||= {}
+      end
+
+      # Number of items from a table
+      #
+      # @param [String] table_name the name of the table
+      #
+      # @return [Integer] the number of items from a table
+      #
+      # @since 0.6.1
+      def count(table_name)
+        table = get_table(table_name)
+        table.items.count
+      end
+    end
+  end
+end