dynomite 1.0.5

data/lib/dynomite/log.rb

@@ -0,0 +1,15 @@
+module Dynomite::Log
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  def log(msg)
+    self.class.log(msg)
+  end
+
+  module ClassMethods
+    def log(msg)
+      Dynomite.logger.info(msg)
+    end
+  end
+end

data/lib/dynomite/migration.rb

@@ -0,0 +1,27 @@
+module Dynomite
+  class Migration
+    autoload :Dsl, "dynomite/migration/dsl"
+    autoload :Generator, "dynomite/migration/generator"
+    autoload :Executor, "dynomite/migration/executor"
+
+    def up
+      puts "Should defined an up method for your migration: #{self.class.name}"
+    end
+
+    def create_table(table_name, &block)
+      execute_with_dsl_params(table_name, :create_table, &block)
+    end
+
+    def update_table(table_name, &block)
+      execute_with_dsl_params(table_name, :update_table, &block)
+    end
+
+  private
+    def execute_with_dsl_params(table_name, method_name, &block)
+      dsl = Dsl.new(method_name, table_name, &block)
+      params = dsl.params
+      executor = Executor.new(table_name, method_name, params)
+      executor.run
+    end
+  end
+end

data/lib/dynomite/migration/common.rb

@@ -0,0 +1,86 @@
+# Common methods to the *SecondaryIndex classes that handle gsi and lsi methods
+# as well a the Dsl class that handles create_table and update_table methods.
+class Dynomite::Migration::Dsl
+  module Common
+    # http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Types/KeySchemaElement.html
+    # partition_key is required
+    def partition_key(identifier)
+      @partition_key_identifier = identifier # for later use. useful for conventional_index_name
+      adjust_schema_and_attributes(identifier, "hash")
+    end
+
+    # sort_key is optional
+    def sort_key(identifier)
+      @sort_key_identifier = identifier # for later use. useful for conventional_index_name
+      adjust_schema_and_attributes(identifier, "range")
+    end
+
+    # Parameters:
+    #   identifier: "id:string" or "id"
+    #   key_type: "hash" or "range"
+    #
+    # Adjusts the parameters for create_table to add the
+    # partition_key and sort_key
+    def adjust_schema_and_attributes(identifier, key_type)
+      name, attribute_type = identifier.split(':')
+      attribute_type = "string" if attribute_type.nil?
+
+      partition_key = {
+        attribute_name: name,
+        key_type: key_type.upcase
+      }
+      @key_schema << partition_key
+
+      attribute_definition = {
+        attribute_name: name,
+        attribute_type: Dynomite::ATTRIBUTE_TYPES[attribute_type]
+      }
+      @attribute_definitions << attribute_definition
+    end
+
+    # t.provisioned_throughput(5) # both
+    # t.provisioned_throughput(:read, 5)
+    # t.provisioned_throughput(:write, 5)
+    # t.provisioned_throughput(:both, 5)
+    def provisioned_throughput(*params)
+      case params.size
+      when 0 # reader method
+        return @provisioned_throughput # early return
+      when 1
+        # @provisioned_throughput_set_called useful for update_table
+        # only provide a provisioned_throughput settings if explicitly called for update_table
+        @provisioned_throughput_set_called = true
+        arg = params[0]
+        if arg.is_a?(Hash)
+          # Case:
+          # provisioned_throughput(
+          #   read_capacity_units: 10,
+          #   write_capacity_units: 10
+          # )
+          @provisioned_throughput = arg # set directly
+          return # early return
+        else # assume parameter is an Integer
+          # Case: provisioned_throughput(10)
+          capacity_type = :both
+          capacity_units = arg
+        end
+      when 2
+        @provisioned_throughput_set_called = true
+        # Case: provisioned_throughput(:read, 5)
+        capacity_type, capacity_units = params
+      end
+
+      map = {
+        read: :read_capacity_units,
+        write: :write_capacity_units,
+      }
+
+      if capacity_type == :both
+        @provisioned_throughput[map[:read]] = capacity_units
+        @provisioned_throughput[map[:write]] = capacity_units
+      else
+        @provisioned_throughput[capacity_type] = capacity_units
+      end
+    end
+  end
+end

data/lib/dynomite/migration/dsl.rb

@@ -0,0 +1,172 @@
+class Dynomite::Migration
+  class Dsl
+    autoload :Common, "dynomite/migration/common"
+    autoload :BaseSecondaryIndex, "dynomite/migration/dsl/base_secondary_index"
+    autoload :LocalSecondaryIndex, "dynomite/migration/dsl/local_secondary_index"
+    autoload :GlobalSecondaryIndex, "dynomite/migration/dsl/global_secondary_index"
+
+    include Dynomite::DbConfig
+    include Common
+
+    attr_accessor :key_schema, :attribute_definitions
+    attr_accessor :table_name
+    def initialize(method_name, table_name, &block)
+      @method_name = method_name
+      @table_name = table_name
+      @block = block
+
+      # Dsl fills in atttributes in as methods are called within the block.
+      # Attributes for both create_table and updated_table:
+      @attribute_definitions = []
+      @provisioned_throughput = {
+        read_capacity_units: 5,
+        write_capacity_units: 5
+      }
+
+      # Attributes for create_table only:
+      @key_schema = []
+
+      # Attributes for update_table only:
+      @gsi_indexes = []
+      @lsi_indexes = []
+    end
+
+    # t.gsi(:create) do |i|
+    #   i.partition_key = "category:string"
+    #   i.sort_key = "created_at:string" # optional
+    # end
+    def gsi(action, index_name=nil, &block)
+      gsi_index = GlobalSecondaryIndex.new(action, index_name, &block)
+      @gsi_indexes << gsi_index # store @gsi_index for the parent Dsl to use
+    end
+    alias_method :global_secondary_index, :gsi
+
+    # t.lsi(:create) do |i|
+    #   i.partition_key = "category:string"
+    #   i.sort_key = "created_at:string" # optional
+    # end
+    def lsi(action=:create, index_name=nil, &block)
+      # dont need action create but have it to keep the lsi and gsi method consistent
+      lsi_index = LocalSecondaryIndex.new(index_name, &block)
+      @lsi_indexes << lsi_index # store @lsi_index for the parent Dsl to use
+    end
+    alias_method :local_secondary_index, :gsi
+
+    def evaluate
+      return if @evaluated
+      @block.call(self) if @block
+      @evaluated = true
+    end
+
+    # http://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/dynamo-example-create-table.html
+    # build the params up from dsl in memory and provides params to the
+      # executor
+    def params
+      evaluate # lazy evaluation: wait until as long as possible before evaluating code block
+
+      # Not using send because think its clearer in this case
+      case @method_name
+      when :create_table
+        params_create_table
+      when :update_table
+        params_update_table
+      end
+    end
+
+    def params_create_table
+      params = {
+        table_name: namespaced_table_name,
+        key_schema: @key_schema,
+        attribute_definitions: @attribute_definitions,
+        provisioned_throughput: @provisioned_throughput
+      }
+
+      params[:local_secondary_index_creates] = local_secondary_index_creates unless @lsi_indexes.empty?
+      params
+    end
+
+    # Goes thorugh all the lsi_indexes that have been built up in memory.
+    # Find the lsi object that creates an index and then grab the
+    # attribute_definitions from it.
+    def lsi_create_attribute_definitions
+      lsi = @lsi_indexes.first # DynamoDB only supports adding one index at a time anyway. The reason @lsi_indexes is an Array is because we're sharing the same class code for LSI and GSI
+      if lsi
+        lsi.evaluate # force early evaluate since we need the params to
+          # add: gsi_attribute_definitions + lsi_attrs
+        lsi_attrs = lsi.attribute_definitions
+      end
+      all_attrs = if lsi_attrs
+                    @attribute_definitions + lsi_attrs
+                  else
+                    @attribute_definitions
+                  end
+      all_attrs.uniq
+    end
+
+    # maps each lsi to the hash structure expected by dynamodb update_table
+    # under the global_secondary_index_updates key:
+    #
+    #   { create: {...} }
+    #   { update: {...} }
+    #   { delete: {...} }
+    def lsi_secondary_index_creates
+      @lsi_indexes.map do |lsi|
+        { lsi.action => lsi.params }
+      end
+    end
+
+    def params_update_table
+      params = {
+        table_name: namespaced_table_name,
+        attribute_definitions: gsi_create_attribute_definitions,
+        # update table take values only some values for the "parent" table
+        # no key_schema, update_table does not handle key_schema for the "parent" table
+      }
+      # only set "parent" table provisioned_throughput if user actually invoked
+      # it in the dsl
+      params[:provisioned_throughput] =  @provisioned_throughput if @provisioned_throughput_set_called
+      params[:global_secondary_index_updates] = global_secondary_index_updates
+      params
+    end
+
+    # Goes thorugh all the gsi_indexes that have been built up in memory.
+    # Find the gsi object that creates an index and then grab the
+    # attribute_definitions from it.
+    def gsi_create_attribute_definitions
+      gsi = @gsi_indexes.find { |gsi| gsi.action == :create }
+      if gsi
+        gsi.evaluate # force early evaluate since we need the params to
+          # add: gsi_attribute_definitions + gsi_attrs
+        gsi_attrs = gsi.attribute_definitions
+      end
+      all_attrs = if gsi_attrs
+                    gsi_attribute_definitions + gsi_attrs
+                  else
+                    gsi_attribute_definitions
+                  end
+      all_attrs.uniq
+    end
+
+    # maps each gsi to the hash structure expected by dynamodb update_table
+    # under the global_secondary_index_updates key:
+    #
+    #   { create: {...} }
+    #   { update: {...} }
+    #   { delete: {...} }
+    def global_secondary_index_updates
+      @gsi_indexes.map do |gsi|
+        { gsi.action => gsi.params }
+      end
+    end
+
+    # >> resp = Post.db.describe_table(table_name: "demo-dev-posts")
+    # >> resp.table.attribute_definitions.map(&:to_h)
+    # => [{:attribute_name=>"id", :attribute_type=>"S"}]
+    def gsi_attribute_definitions
+      return @gsi_attribute_definitions if @gsi_attribute_definitions
+
+      resp = db.describe_table(table_name: namespaced_table_name)
+      @gsi_attribute_definitions = resp.table.attribute_definitions.map(&:to_h)
+    end
+  end
+end

data/lib/dynomite/migration/dsl/base_secondary_index.rb

@@ -0,0 +1,72 @@
+# Base class for LocalSecondaryIndex and GlobalSecondaryIndex
+class Dynomite::Migration::Dsl
+  class BaseSecondaryIndex
+    include Common
+
+    attr_accessor :action, :key_schema, :attribute_definitions
+    attr_accessor :index_name
+    def initialize(action, index_name=nil, &block)
+      @action = action.to_sym
+      # for gsi action can be: :create, :update, :delete
+      # for lsi action is always: :create
+      @index_name = index_name
+      @block = block
+
+      # Dsl fills these atttributes in as methods are called within
+      # the block
+      @key_schema = []
+      @attribute_definitions = []
+      # default provisioned_throughput
+      @provisioned_throughput = {
+        read_capacity_units: 5,
+        write_capacity_units: 5
+      }
+    end
+
+    def index_name
+      @index_name || conventional_index_name
+    end
+
+    def conventional_index_name
+      # @partition_key_identifier and @sort_key_identifier are set as immediately
+      # when the partition_key and sort_key methods are called in the dsl block.
+      # Usually look like this:
+      #
+      #   @partition_key_identifier: post_id:string
+      #   @sort_key_identifier: updated_at:string
+      #
+      # We strip the :string portion in case it is provided
+      #
+      partition_key = @partition_key_identifier.split(':').first
+      sort_key = @sort_key_identifier.split(':').first if @sort_key_identifier
+      [partition_key, sort_key, "index"].compact.join('-')
+    end
+
+    def evaluate
+      return if @evaluated
+      @block.call(self) if @block
+      @evaluated = true
+    end
+
+    def params
+      evaluate # lazy evaluation: wait until as long as possible before evaluating code block
+
+      params = { index_name: index_name } # required for all actions
+
+      if @action == :create
+        params[:key_schema] = @key_schema # required for create action
+        # hardcode to ALL for now
+        params[:projection] = { # required
+          projection_type: "ALL", # accepts ALL, KEYS_ONLY, INCLUDE
+          # non_key_attributes: ["NonKeyAttributeName"],
+        }
+      end
+
+      if [:create, :update].include?(@action)
+        params[:provisioned_throughput] = @provisioned_throughput
+      end
+
+      params
+    end
+  end
+end

data/lib/dynomite/migration/dsl/global_secondary_index.rb

@@ -0,0 +1,4 @@
+class Dynomite::Migration::Dsl
+  class GlobalSecondaryIndex < BaseSecondaryIndex
+  end
+end

data/lib/dynomite/migration/dsl/local_secondary_index.rb

@@ -0,0 +1,8 @@
+class Dynomite::Migration::Dsl
+  class LocalSecondaryIndex < BaseSecondaryIndex
+    def initialize(index_name=nil, &block)
+      # Can only create local secondary index when creating a table
+      super(:create, index_name, &block)
+    end
+  end
+end

data/lib/dynomite/migration/executor.rb

@@ -0,0 +1,30 @@
+class Dynomite::Migration
+  class Executor
+    include Dynomite::DbConfig
+
+    # Examples:
+    #  Executor.new(:create_table, params) or
+    #  Executor.new(:update_table, params)
+    #
+    # The params are generated frmo the dsl.params
+    attr_accessor :table_name
+    def initialize(table_name, method_name, params)
+      @table_name = table_name
+      @method_name = method_name # create_table or update_table
+      @params = params
+    end
+
+    def run
+      begin
+        # Examples:
+        #   result = db.create_table(@params)
+        #   result = db.update_table(@params)
+        result = db.send(@method_name, @params)
+
+        puts "DynamoDB Table: #{@table_name} Status: #{result.table_description.table_status}"
+      rescue Aws::DynamoDB::Errors::ServiceError => error
+        puts "Unable to #{@method_name.to_s.gsub('_',' ')}: #{error.message}"
+      end
+    end
+  end
+end

data/lib/dynomite/migration/generator.rb

@@ -0,0 +1,68 @@
+require "active_support/core_ext/string"
+
+class Dynomite::Migration
+  # jets dynamodb:generate posts --partition-key id:string
+  class Generator
+    include Dynomite::DbConfig
+
+    attr_reader :migration_name, :table_name
+    def initialize(migration_name, options)
+      @migration_name = migration_name
+      @options = options
+    end
+
+    def generate
+      puts "Generating migration for #{@table_name}" unless @options[:quiet]
+      return if @options[:noop]
+      create_migration
+    end
+
+    def create_migration
+      FileUtils.mkdir_p(File.dirname(migration_path))
+      IO.write(migration_path, migration_code)
+      puts "Migration file created: #{migration_path}. \nTo run:"
+      puts "  jets dynamodb:migrate #{migration_path}"
+    end
+
+    def migration_code
+      path = File.expand_path("../templates/#{table_action}.rb", __FILE__)
+      result = Dynomite::Erb.result(path,
+        migration_class_name: migration_class_name,
+        table_name: table_name,
+        partition_key: @options[:partition_key],
+        sort_key: @options[:sort_key],
+        provisioned_throughput: @options[:provisioned_throughput] || 5,
+      )
+    end
+
+    def table_action
+      @options[:table_action] || conventional_table_action
+    end
+
+    def conventional_table_action
+      @migration_name.include?("update") ? "update_table" : "create_table"
+    end
+
+    def table_name
+      @options[:table_name] || conventional_table_name
+    end
+
+    # create_posts => posts
+    # update_posts => posts
+    def conventional_table_name
+      @migration_name.sub(/^(create|update)_/, '')
+    end
+
+    def migration_class_name
+      "#{@migration_name}_migration".classify # doesnt include timestamp
+    end
+
+    def migration_path
+      "#{Dynomite.app_root}dynamodb/migrate/#{timestamp}-#{@migration_name}_migration.rb"
+    end
+
+    def timestamp
+      @timestamp ||= Time.now.strftime("%Y%m%d%H%M%S")
+    end
+  end
+end