toy-dynamo 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c505b06ca1b99d16eb355a403fa7dcc40b9653a
-  data.tar.gz: 010db3bf334c8759fc922cd5801b235b76cf3f05
+  metadata.gz: 342640e00f6ef382f4ad50e46015753ca5ac034f
+  data.tar.gz: 27ae95e381c9ca143507c0969edde91f9c9b51ee
 SHA512:
-  metadata.gz: 7c76caefca0f5ecb58c5230afa9940b9e8efbc1c3c840d788f04296c4a40afdb064c3fd843a8dab837fedf3a4e7b0f4c89359ac4615a9139f791ef65afae947a
-  data.tar.gz: 1a48d0ef0c3efd90e26bb154850500ca462bc229b2fb6328522d2a5a453970e1400ce67a7f8cd3b632b875e04560d1e65cb99c6bce072c29d813b8ca9a4ad015
+  metadata.gz: 5b7857e1165a65db0c8e0b61e12539eac7106cc9acdb34637b558abd652f1632161b8c1e676f6234e090d659b7d4ba3eeecede3d82ea286260ff868a32b86b70
+  data.tar.gz: 372cb2202e35588fad3aa8ed830dce5a12b654880f27e143d3cf782816b4e321a6cb77db960b571be54eb82640d23c4a42b2c0e3581c2807c0aa2b539faa4e2d

data/README.md

@@ -1,3 +1,10 @@
+## Overview
+toy-dynamo is an ORM for AWS DynamoDB.  It is an extension to [toystore](https://github.com/jnunemaker/toystore) that provides an ActiveModel based ORM for schema-less type data stores.
+
+> **Toy::Object** comes with all the goods you need for plain old ruby objects -- attributes, dirty attribute tracking, equality, inheritance, serialization, cloning, logging and pretty inspecting.
+
+> **Toy::Store** includes Toy::Object and adds identity, persistence and querying through adapters, mass assignment, callbacks, validations and a few simple associations (lists and references).
+
 ## Install
 * toystore gem
 	* originally [https://github.com/jnunemaker/toystore](https://github.com/jnunemaker/toystore)
@@ -8,12 +15,29 @@
 In ActiveModel model:
 
 ```
-dynamo_table do
-	adapter :dynamo, AWS::DynamoDB::ClientV2.new, {:model => self}
-	hash_key :thread_guid
+class Comment
+
+	include Toy::Dynamo
+
+	adapter :dynamo, Toy::Dynamo::Adapter.default_client, {:model => self}
+
+	dynamo_table do
+		hash_key :thread_guid
+		range_key :comment_guid
+		local_secondary_index :posted_by
+		read_provision 50
+    write_provision 10
+	end
+	
+	attribute :thread_guid, String
+	attribute :comment_guid, String, :default => lambda { SimpleUUID::UUID.new.to_guid }
+	attribute :body, String
+	attribute :posted_by, String
+	
 end
 ```
-* **Other options:**
+* **Other options for 'dynamo_table' config:**
+	* hash_key :comment_id
 	* range_key :comment_id
 	* table_name "user_dynamo_table"
 	* read_provision 20
@@ -23,8 +47,11 @@ end
 		* :projection => :all
 		* Can also specify an Array of attributes to project besides the primary keys ex:
 			* :projection => [:subject, :commenter_name]
+* **aws-sdk config:**
+  * [http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/Core/Configuration.html](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/Core/Configuration.html)
 
-## Basic Usage
+
+## Basic Usage (toystore)
 * **Read Hash Key Only**
 	* Example:
 		* Model.read("xyz")
@@ -68,6 +95,15 @@ end
       :dynamo_db_endpoint => 'localhost',
       :dynamo_db_port => 4567
     }), {:model => self}
+* **Create table**
+	* rake ddb:create CLASS=User
+	* rake ddb:create CLASS=User TABLE=user-2013-03-14
+* **Delete table**
+	* rake ddb:delete CLASS=User
+	* rake ddb:delete CLASS=User TABLE=user-2013-03-14
+* **Resize table read/write capacity**
+  * rake ddb:resize CLASS=User READ=50 WRITE=10
+  * rake ddb:resize CLASS=User # Use values from model dynamo_table read_provision/write_provision
 
 ## Compatibility
 * Tested with
@@ -76,6 +112,7 @@ end
 
 ## TODO
 * raise error if trying to use an attribute that wasn't 'select'ed (defaulting to selecting all attributes which loads everything with an extra read)
-* while loop for situation where batch_get_item returns batched results
-* error out on mismatch of table schema from dynamo_table schema (changed?)
-* allow changes to read write capacity
+* lambdas for table_name (for dynamic table names)
+* string and number sets (mostly for scans)
+* use MAX_ITEM_SIZE (64kb)
+

data/lib/toy/dynamo/adapter.rb

@@ -3,6 +3,21 @@ require 'adapter'
 module Toy
   module Dynamo
     module Adapter
+      
+      def self.default_client(config={})
+        options={}
+        options[:use_ssl] = Toy::Dynamo::Config.use_ssl
+        options[:use_ssl] = config[:use_ssl] if config.has_key?(:use_ssl)
+        options[:dynamo_db_endpoint] = config[:endpoint] || Toy::Dynamo::Config.endpoint
+        options[:dynamo_db_port] = config[:port] || Toy::Dynamo::Config.port
+
+        options[:dynamo_db_endpoint] = config[:endpoint] || Toy::Dynamo::Config.endpoint
+        options[:dynamo_db_port] = config[:port] || Toy::Dynamo::Config.port
+        #:dynamo_db_crc_32_check = false
+
+        @@default_client ||= AWS::DynamoDB::ClientV2.new(options)
+      end
+
       def read(key, options=nil)
         options ||= {}
         attrs = nil

data/lib/toy/dynamo/config/options.rb

@@ -0,0 +1,80 @@
+# Shamelessly stolen from Mongoid (and Dynamoid)!
+module Toy #:nodoc
+  module Dynamo #:nodoc
+    module Config
+
+      # Encapsulates logic for setting options.
+      module Options
+
+        # Get the defaults or initialize a new empty hash.
+        #
+        # @example Get the defaults.
+        #   options.defaults
+        #
+        # @return [ Hash ] The default options.
+        #
+        # @since 0.2.0
+        def defaults
+          @defaults ||= {}
+        end
+
+        # Define a configuration option with a default.
+        #
+        # @example Define the option.
+        #   Options.option(:persist_in_safe_mode, :default => false)
+        #
+        # @param [ Symbol ] name The name of the configuration option.
+        # @param [ Hash ] options Extras for the option.
+        #
+        # @option options [ Object ] :default The default value.
+        #
+        # @since 0.2.0
+        def option(name, options = {})
+          defaults[name] = settings[name] = options[:default]
+
+          class_eval <<-RUBY
+            def #{name}
+              settings[#{name.inspect}]
+            end
+
+            def #{name}=(value)
+              settings[#{name.inspect}] = value
+            end
+
+            def #{name}?
+              #{name}
+            end
+            
+            def reset_#{name}
+              settings[#{name.inspect}] = defaults[#{name.inspect}]
+            end
+          RUBY
+        end
+
+        # Reset the configuration options to the defaults.
+        #
+        # @example Reset the configuration options.
+        #   config.reset
+        #
+        # @return [ Hash ] The defaults.
+        #
+        # @since 0.2.0
+        def reset
+          settings.replace(defaults)
+        end
+
+        # Get the settings or initialize a new empty hash.
+        #
+        # @example Get the settings.
+        #   options.settings
+        #
+        # @return [ Hash ] The setting options.
+        #
+        # @since 0.2.0
+        def settings
+          @settings ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/toy/dynamo/config.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+require "uri"
+require "toy/dynamo/config/options"
+
+# Shamelessly stolen from Dynamoid
+module Toy
+  module Dynamo
+
+    # Contains all the basic configuration information required for Dynamoid: both sensible defaults and required fields.
+    module Config
+      extend self
+      extend Options
+
+      # All the default options.
+      option :logger, :default => defined?(Rails)
+      option :read_provision, :default => 50
+      option :write_provision, :default => 10
+      option :included_models, :default => []
+      # TODO - default adapter client based on config
+      #option :namespace, :default => defined?(Rails) ? "#{Rails.application.class.parent_name}_#{Rails.env}" : ""
+      option :endpoint, :default => 'dynamodb.us-west-2.amazonaws.com'
+      option :port, :default => 443
+      option :use_ssl, :default => true
+      # Use AWS config
+      #option :access_key
+      #option :secret_key
+
+      # The default logger for Dynamoid: either the Rails logger or just stdout.
+      #
+      # @since 0.2.0
+      def default_logger
+        defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : ::Logger.new($stdout)
+      end
+
+      # Returns the assigned logger instance.
+      #
+      # @since 0.2.0
+      def logger
+        @logger ||= default_logger
+      end
+
+      # If you want to, set the logger manually to any output you'd like. Or pass false or nil to disable logging entirely.
+      #
+      # @since 0.2.0
+      def logger=(logger)
+        case logger
+        when false, nil then @logger = nil
+        when true then @logger = default_logger
+        else
+          @logger = logger if logger.respond_to?(:info)
+        end
+      end
+
+    end
+  end
+end

data/lib/toy/dynamo/querying.rb

@@ -4,24 +4,25 @@ module Toy
       extend ActiveSupport::Concern
 
       # Failsafe
-      MAX_BATCH_ITERATIONS = 100
+      MAX_BATCH_ITERATIONS = 1000
+      DEFAULT_BATCH_SIZE = 100
 
       module ClassMethods
 
         # Read results up to the limit
-        #   read_range("1", :range_value => "2", :limit => 10)
+        #   read_range("1", :range => { :varname.gte => "2"}, :limit => 10)
         # Loop results in given batch size until limit is hit or no more results
-        #   read_range("1", :range_value => "2", :batch => 10, :limit => 1000)
+        #   read_range("1", :range => { :varname.eq => "2"}, :batch => 10, :limit => 1000)
         def read_range(hash_value, options={})
           raise ArgumentError, "no range_key specified for this table" if dynamo_table.range_keys.blank?
           aggregated_results = []
 
-          if (batch_size = options.delete(:batch))
-            max_results_limit = options[:limit]
-            if options[:limit] && options[:limit] > batch_size
-              options.merge!(:limit => batch_size)
-            end
+          batch_size = options.delete(:batch) || DEFAULT_BATCH_SIZE
+          max_results_limit = options[:limit]
+          if options[:limit] && options[:limit] > batch_size
+            options.merge!(:limit => batch_size)
           end
+
           results = dynamo_table.query(hash_value, options)
           response = Response.new(results)
 
@@ -30,7 +31,7 @@ module Toy
             aggregated_results << load(attrs[dynamo_table.hash_key[:attribute_name]], attrs)
           end
 
-          if batch_size
+          if response.more_results?
             results_returned = response.count
             batch_iteration = 0
             while response.more_results? && batch_iteration < MAX_BATCH_ITERATIONS

data/lib/toy/dynamo/schema.rb

@@ -60,7 +60,7 @@ module Toy
             raise(ArgumentError, "Invalid read provision") unless val.to_i >= 1
             @dynamo_read_provision = val.to_i
           else
-            @dynamo_read_provision || 10
+            @dynamo_read_provision || Toy::Dynamo::Config.read_provision
           end
         end
 
@@ -69,7 +69,7 @@ module Toy
             raise(ArgumentError, "Invalid write provision") unless val.to_i >= 1
             @dynamo_write_provision = val.to_i
           else
-            @dynamo_write_provision || 10
+            @dynamo_write_provision || Toy::Dynamo::Config.write_provision
           end
         end
 
@@ -162,24 +162,24 @@ module Toy
         end
 
         def validate_key_schema
-          if (@dynamo_table.schema_loaded_from_dynamo[:table][:key_schema] != table_schema[:key_schema])
+          if (@dynamo_table.schema_loaded_from_dynamo[:table][:key_schema].sort_by { |k| k[:key_type] } != table_schema[:key_schema].sort_by { |k| k[:key_type] })
             raise ArgumentError, "It appears your key schema (Hash Key/Range Key) have changed from the table definition. Rebuilding the table is necessary."
           end
 
-          if (@dynamo_table.schema_loaded_from_dynamo[:table][:attribute_definitions] != table_schema[:attribute_definitions])
+          if (@dynamo_table.schema_loaded_from_dynamo[:table][:attribute_definitions].sort_by { |k| k[:attribute_name] } != table_schema[:attribute_definitions].sort_by { |k| k[:attribute_name] })
             raise ArgumentError, "It appears your attribute definition (types?) have changed from the table definition. Rebuilding the table is necessary."
           end
           
-          if (@dynamo_table.schema_loaded_from_dynamo[:table][:local_secondary_indexes].dup.collect {|i| i.delete_if{|k, v| [:index_size_bytes, :item_count].include?(k) }; i } != table_schema[:local_secondary_indexes])
+          if (@dynamo_table.schema_loaded_from_dynamo[:table][:local_secondary_indexes].dup.collect {|i| i.delete_if{|k, v| [:index_size_bytes, :item_count].include?(k) }; i }.sort_by { |lsi| lsi[:index_name] } != table_schema[:local_secondary_indexes].sort_by { |lsi| lsi[:index_name] })
             raise ArgumentError, "It appears your local secondary indexes have changed from the table definition. Rebuilding the table is necessary."
           end
 
           if @dynamo_table.schema_loaded_from_dynamo[:table][:provisioned_throughput][:read_capacity_units] != read_provision
-            puts "read_capacity_units mismatch. Need to update table?"
+            Toy::Dynamo::Config.logger.error "read_capacity_units mismatch. Need to update table?"
           end
 
           if @dynamo_table.schema_loaded_from_dynamo[:table][:provisioned_throughput][:write_capacity_units] != write_provision
-            puts "write_capacity_units mismatch. Need to update table?"
+            Toy::Dynamo::Config.logger.error "write_capacity_units mismatch. Need to update table?"
           end
         end
 

data/lib/toy/dynamo/table.rb

@@ -40,7 +40,7 @@ module Toy
         begin
           self.load_schema
         rescue AWS::DynamoDB::Errors::ResourceNotFoundException => e
-          puts "No table found!"
+          Toy::Dynamo::Config.logger.error "No table found!"
           #self.create
           #self.load_schema
         end
@@ -105,7 +105,7 @@ module Toy
         options[:select] ||= []
 
         get_item_request = {
-          :table_name => options[:table_name] || @table_schema[:table_name],
+          :table_name => options[:table_name] || self.table_name,
           :key => hash_key_item_param(hash_key),
           :consistent_read => options[:consistent_read],
           :return_consumed_capacity => RETURNED_CONSUMED_CAPACITY[options[:return_consumed_capacity]]
@@ -133,7 +133,7 @@ module Toy
         key_conditions.merge!(hash_key_condition_param(hash_key_value))
 
         query_request = {
-          :table_name => options[:table_name] || @table_schema[:table_name],
+          :table_name => options[:table_name] || self.table_name,
           :key_conditions => key_conditions,
           :consistent_read => options[:consistent_read],
           :return_consumed_capacity => RETURNED_CONSUMED_CAPACITY[options[:return_consumed_capacity]],
@@ -222,7 +222,7 @@ module Toy
         request_items_request.merge!( :attributes_to_get => [options[:select]].flatten ) unless options[:select].blank?
         request_items_request.merge!( :consistent_read => options[:consistent_read] ) if options[:consistent_read]
         batch_get_item_request = {
-          :request_items => { (options[:table_name] || @table_schema[:table_name]) => request_items_request },
+          :request_items => { (options[:table_name] || self.table_name) => request_items_request },
           :return_consumed_capacity => RETURNED_CONSUMED_CAPACITY[options[:return_consumed_capacity]]
         }
         @client.batch_get_item(batch_get_item_request)
@@ -260,7 +260,7 @@ module Toy
             })
           end
           update_item_request = {
-            :table_name => options[:table_name] || @table_schema[:table_name],
+            :table_name => options[:table_name] || self.table_name,
             :key => key_request,
             :attribute_updates => attrs_to_update,
             :return_consumed_capacity => RETURNED_CONSUMED_CAPACITY[options[:return_consumed_capacity]]
@@ -274,7 +274,7 @@ module Toy
           end
           items.merge!(hash_key_item_param(hash_key_value))
           put_item_request = {
-            :table_name => options[:table_name] || @table_schema[:table_name],
+            :table_name => options[:table_name] || self.table_name,
             :item => items,
             :return_consumed_capacity => RETURNED_CONSUMED_CAPACITY[options[:return_consumed_capacity]]
           }
@@ -298,7 +298,7 @@ module Toy
           })
         end
         delete_item_request = {
-          :table_name => options[:table_name] || @table_schema[:table_name],
+          :table_name => options[:table_name] || self.table_name,
           :key => key_request
         }
         @client.delete_item(delete_item_request)
@@ -324,13 +324,22 @@ module Toy
         end
       end
 
+      # Call proc or return string
+      def table_name
+        if @table_schema[:table_name].respond_to?(:call)
+          @table_schema[:table_name].call
+        else
+          @table_schema[:table_name]
+        end
+      end
+
       def create(options={})
-        if @client.list_tables[:table_names].include?(options[:table_name] || @table_schema[:table_name])
-          raise "Table #{options[:table_name] || @table_schema[:table_name]} already exists!"
+        if @client.list_tables[:table_names].include?(options[:table_name] || self.table_name)
+          raise "Table #{options[:table_name] || self.table_name} already exists"
         end
 
         @client.create_table(@table_schema.merge({
-          :table_name => options[:table_name] || @table_schema[:table_name]
+          :table_name => options[:table_name] || self.table_name
         }))
 
         while (table_metadata = self.describe(options))[:table][:table_status] == "CREATING"
@@ -340,19 +349,35 @@ module Toy
       end
 
       def describe(options={})
-        @client.describe_table(:table_name => options[:table_name] || @table_schema[:table_name])
+        @client.describe_table(:table_name => options[:table_name] || self.table_name)
       end
 
       def delete(options={})
-        return false unless @client.list_tables[:table_names].include?(options[:table_name] || @table_schema[:table_name])
-        @client.delete_table(:table_name => options[:table_name] || @table_schema[:table_name])
+        return false unless @client.list_tables[:table_names].include?(options[:table_name] || self.table_name)
+        @client.delete_table(:table_name => options[:table_name] || self.table_name)
         begin
           while (table_metadata = self.describe) && table_metadata[:table][:table_status] == "DELETING"
             sleep 1
           end
         rescue AWS::DynamoDB::Errors::ResourceNotFoundException => e
-          puts "Table deleted!"
+          Toy::Dynamo::Config.logger.info "Table deleted"
+        end
+        true
+      end
+
+      def resize(options={})
+        return false unless @client.list_tables[:table_names].include?(options[:table_name] || self.table_name)
+        @client.update_table({
+          :provisioned_throughput => {
+            :read_capacity_units => (options[:read_capacity_units] || @table_schema[:provisioned_throughput][:read_capacity_units]).to_i,
+            :write_capacity_units => (options[:write_capacity_units] || @table_schema[:provisioned_throughput][:write_capacity_units]).to_i
+          },
+          :table_name => options[:table_name] || self.table_name
+        })
+        while (table_metadata = self.describe) && table_metadata[:table][:table_status] == "UPDATING"
+          sleep 1
         end
+        Toy::Dynamo::Config.logger.info "Table resized to #{table_metadata[:table][:provisioned_throughput]}"
         true
       end
 

data/lib/toy/dynamo/tasks.rb

@@ -9,6 +9,16 @@ namespace :ddb do
     ENV['CLASS'].constantize.dynamo_table.create(options)
   end
 
+  desc 'Resize a DynamoDB table read/write provision'
+  task :resize => :environment do
+    raise "expected usage: rake ddb:resize CLASS=User" unless ENV['CLASS']
+    options = {}
+    options.merge!(:table_name => ENV['TABLE']) if ENV['TABLE']
+    options.merge!(:read_capacity_units => ENV['READ'].to_i) if ENV['READ']
+    options.merge!(:write_capacity_units => ENV['WRITE'].to_i) if ENV['WRITE']
+    ENV['CLASS'].constantize.dynamo_table.resize(options)
+  end
+
   task :destroy => :environment do
     raise "expected usage: rake ddb:destroy CLASS=User" unless ENV['CLASS']
     options = {}

data/lib/toy/dynamo/version.rb

@@ -1,5 +1,5 @@
 module Toy
   module Dynamo
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end

data/lib/toy/dynamo.rb

@@ -8,6 +8,7 @@ require "toy/dynamo/response"
 require "toy/dynamo/persistence"
 # Override 'write_attribute' for hash_key == id
 require "toy/dynamo/attributes"
+require "toy/dynamo/config"
 require "toy/dynamo/extensions/array"
 require "toy/dynamo/extensions/boolean"
 require "toy/dynamo/extensions/date"
@@ -18,6 +19,9 @@ require "toy/dynamo/extensions/symbol"
 
 module Toy
   module Dynamo
+
+    MAX_ITEM_SIZE = 65_536
+
     extend ActiveSupport::Concern
     include Toy::Store
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toy-dynamo
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Cary Dunn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-17 00:00:00.000000000 Z
+date: 2013-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,6 +81,8 @@ files:
 - lib/toy/dynamo.rb
 - lib/toy/dynamo/adapter.rb
 - lib/toy/dynamo/attributes.rb
+- lib/toy/dynamo/config.rb
+- lib/toy/dynamo/config/options.rb
 - lib/toy/dynamo/extensions/array.rb
 - lib/toy/dynamo/extensions/boolean.rb
 - lib/toy/dynamo/extensions/date.rb