dynomite 1.2.7 → 2.0.0

data/lib/dynomite/migration/dsl/index/base.rb

@@ -0,0 +1,42 @@
+module Dynomite::Migration::Dsl::Index
+  class Base
+    include Dynomite::Types
+
+    def partition_key_attribute_name
+      get_attribute_name(:partition_key)
+    end
+
+    def sort_key_attribute_name
+      get_attribute_name(:sort_key)
+    end
+
+    def get_attribute_name(field=:partition_key)
+      value = instance_variable_get("@#{field}") # IE: @partition_key
+      value.to_s.split(':').first if value
+    end
+
+    def partition_key_attribute_type
+      get_attribute_type(:partition_key)
+    end
+
+    def sort_key_attribute_type
+      get_attribute_type(:sort_key)
+    end
+
+    def get_attribute_type(field=:partition_key)
+      value = instance_variable_get("@#{field}") # IE: @partition_key
+      name, type = value.to_s.split(':')
+      type ||= "string"
+      type_map(type)
+    end
+
+    def conventional_index_name
+      # DynamoDB requires index names to be at least 3 characters long, otherwise:
+      #   Error: Unable to : 1 validation error detected: Value 'id' at 'globalSecondaryIndexes.1.member.indexName' failed to satisfy constraint: Member must have length greater than or equal to 3
+      # The id valid is too short sadly.
+      # Adding -index to the end of the index name is a safe way to ensure that.
+      # Annoying that the index_name("id-index") is going to be a little longer.
+      [partition_key_attribute_name, sort_key_attribute_name, 'index'].compact.join('-')
+    end
+  end
+end

data/lib/dynomite/migration/dsl/index/gsi.rb

@@ -0,0 +1,59 @@
+module Dynomite::Migration::Dsl::Index
+  class Gsi < Base
+    attr_reader :action, :attrs
+    def initialize(action, attrs)
+      @action = action
+      @params = attrs.dup
+      # Delete the special DSL keys. Keep the rest and pass through to AWS create_table or update_table
+      @partition_key = @params.delete(:partition_key) || @params.delete(:hash_key) # required for create, optional for update (only index needed)
+      @sort_key = @params.delete(:sort_key) || @params.delete(:range_key)          # optional for create
+    end
+
+    def params
+      send("params_#{@action}")
+    end
+
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#create_table-instance_method
+    def params_create
+      @params[:key_schema] ||= []
+      # HASH - add to beginning of array
+      @params[:key_schema].unshift(attribute_name: partition_key_attribute_name, key_type: "HASH") unless @partition_key.blank?
+      # RANGE - add to end of array
+      @params[:key_schema] << {attribute_name: sort_key_attribute_name, key_type: "RANGE"} unless @sort_key.blank?
+      @params[:index_name] ||= conventional_index_name
+      @params[:projection] ||= {projection_type: "ALL"}
+      @params[:provisioned_throughput] = normalize_provisioned_throughput(@params[:provisioned_throughput]) if @params[:provisioned_throughput]
+      @params
+    end
+
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#update_table-instance_method
+    def params_update
+      @params[:index_name] ||= conventional_index_name
+      @params[:provisioned_throughput] = normalize_provisioned_throughput(@params[:provisioned_throughput]) if @params[:provisioned_throughput]
+      @params
+    end
+
+    def params_delete
+      @params[:index_name] ||= conventional_index_name
+      @params
+    end
+
+    def attribute_definitions
+      definitions = []
+      definitions << {attribute_name: partition_key_attribute_name, attribute_type: partition_key_attribute_type} unless @partition_key.blank?
+      definitions << {attribute_name: sort_key_attribute_name, attribute_type: sort_key_attribute_type} unless @sort_key.blank?
+      definitions
+    end
+
+    def normalize_provisioned_throughput(value)
+      if value.is_a?(Hash)
+        value
+      else
+        {
+          read_capacity_units: value,
+          write_capacity_units: value,
+        }
+      end
+    end
+  end
+end

data/lib/dynomite/migration/dsl/index/lsi.rb

@@ -0,0 +1,27 @@
+module Dynomite::Migration::Dsl::Index
+  class Lsi < Base
+    attr_reader :attrs
+    def initialize(partition_key_name, attrs)
+      @partition_key_name = partition_key_name
+      @params = attrs.dup
+      # Delete the special DSL keys. Keep the rest and pass through to AWS create_table or update_table
+      @sort_key = @params.delete(:sort_key) || @params.delete(:range_key) # require for LSI index since it must be a composite key
+    end
+
+    def params
+      @params[:key_schema] ||= []
+      @params[:key_schema] << {attribute_name: @partition_key_name, key_type: "HASH"}
+      @params[:key_schema] << {attribute_name: sort_key_attribute_name, key_type: "RANGE"}
+      @params[:index_name] ||= conventional_index_name
+      @params[:projection] ||= {projection_type: "ALL"}
+      @params
+    end
+
+    def attribute_definitions
+      definitions = []
+      definitions << {attribute_name: partition_key_attribute_name, attribute_type: partition_key_attribute_type} unless @partition_key.blank?
+      definitions << {attribute_name: sort_key_attribute_name, attribute_type: sort_key_attribute_type} unless @sort_key.blank?
+      definitions
+    end
+  end
+end

data/lib/dynomite/migration/dsl/index.rb

@@ -0,0 +1,72 @@
+class Dynomite::Migration::Dsl
+  module Index
+    def add_gsi(attrs={})
+      attrs = normalize_index_attrs(attrs, :partition_key)
+      @gsi_indexes << Gsi.new(:create, attrs) # store in @gsi_indexes for the parent Dsl to use
+    end
+    alias create_gsi add_gsi
+
+    def update_gsi(attrs={})
+      attrs = normalize_index_attrs(attrs, :partition_key)
+      @gsi_indexes << Gsi.new(:update, attrs) # store in @gsi_indexes for the parent Dsl to use
+    end
+
+    def remove_gsi(attrs={})
+      attrs = normalize_index_attrs(attrs, :partition_key)
+      @gsi_indexes << Gsi.new(:delete, attrs) # store in @gsi_indexes for the parent Dsl to use
+    end
+    alias delete_gsi remove_gsi
+
+    def add_lsi(attrs={})
+      # dont need create with lsi indexes, they are created as part of create_table
+      partition_key_name = @partition_key_identifier.split(':').first
+      attrs = normalize_index_attrs(attrs, :sort_key)
+      @lsi_indexes << Lsi.new(partition_key_name, attrs) # store in @lsi_indexes for the parent Dsl to use
+    end
+    alias create_lsi add_lsi
+
+    def normalize_index_attrs(attrs, type=:partition_key)
+      if attrs.is_a?(String) || attrs.is_a?(Symbol)
+        {type => attrs.to_s}
+      else
+        attrs
+      end
+    end
+
+    # maps each lsi to the hash structure expected by dynamodb update_table
+    # under the lsi_secondary_index_creates key:
+    #
+    #   { create: {...} }
+    #   { update: {...} }
+    #   { delete: {...} }
+    def lsi_secondary_index_creates
+      @lsi_indexes.map do |lsi|
+        lsi.params
+      end
+    end
+
+    # maps each lsi to the hash structure expected by dynamodb update_table
+    # under the gsi_secondary_index_creates key:
+    #
+    #   { create: {...} }
+    #   { update: {...} }
+    #   { delete: {...} }
+    def gsi_secondary_index_creates
+      @gsi_indexes.map do |gsi|
+        gsi.params
+      end
+    end
+
+    # maps each gsi to the hash structure expected by dynamodb update_table
+    # under the global_secondary_index_updates key:
+    #
+    #   { create: {...} }
+    #   { update: {...} }
+    #   { delete: {...} }
+    def global_secondary_index_updates
+      @gsi_indexes.map do |gsi|
+        { gsi.action => gsi.params }
+      end
+    end
+  end
+end

data/lib/dynomite/migration/dsl/primary_key.rb

@@ -0,0 +1,62 @@
+# Common methods to the *SecondaryIndex classes that handle gsi and lsi methods
+# as well a the Dsl class that handles create_table and update_table methods.
+class Dynomite::Migration::Dsl
+  module PrimaryKey
+    include Dynomite::Types
+
+    # http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Types/KeySchemaElement.html
+    # partition_key is required
+    attr_reader :partition_key_identifier
+    def partition_key(identifier)
+      identifier = identifier.to_s
+      @partition_key_identifier = identifier # for later use. useful for conventional_index_name
+      adjust_schema_and_attributes(identifier, "HASH")
+    end
+
+    # sort_key is optional
+    attr_reader :sort_key_identifier
+    def sort_key(identifier)
+      identifier = identifier.to_s
+      @sort_key_identifier = identifier # for later use. useful for conventional_index_name
+      adjust_schema_and_attributes(identifier, "RANGE")
+    end
+
+    # Use class variable to share across all instances of GlobalSecondaryIndex
+    # Note: Even though dynomite merges attribute defintions correctly for
+    # multiple indexes, DynamoDB does not allow creating multiple indexes
+    # at the same time. Example error:
+    #   Unable to update table: Subscriber limit exceeded: Only 1 online index can be created or deleted simultaneously per table
+    # Leaving the logic in place in case AWS changes this in the future.
+
+    # Parameters:
+    #   identifier: "id:string" or "id"
+    #   key_type: "HASH" OR "RANGE"
+    #
+    # Adjusts the parameters for create_table to add the
+    # partition_key and sort_key
+    def adjust_schema_and_attributes(identifier, key_type)
+      name, attribute_type = identifier.split(':')
+      attribute_type ||= "string" # default to string
+
+      partition_key = {
+        attribute_name: name,
+        key_type: key_type.upcase
+      }
+
+      if partition_key[:key_type] == "RANGE"
+        @key_schema << partition_key unless @key_schema.include?(partition_key)
+      else # HASH - add to beginning
+        @key_schema.unshift(partition_key) unless @key_schema.include?(partition_key)
+      end
+
+      attribute_definition = {
+        attribute_name: name,
+        attribute_type: type_map(attribute_type)
+      }
+      unless @attribute_definitions.include?(attribute_definition)
+        @attribute_definitions << attribute_definition
+      end
+      @attribute_definitions
+    end
+  end
+end

data/lib/dynomite/migration/dsl/provisioned_throughput.rb

@@ -0,0 +1,38 @@
+class Dynomite::Migration::Dsl
+  module ProvisionedThroughput
+    def billing_mode(v)
+      @billing_mode = v
+      @provisioned_throughput = nil if @billing_mode.to_s.upcase == "PAY_PER_REQUEST"
+    end
+
+    # t.provisioned_throughput(1) # both
+    # t.provisioned_throughput(1,1) # read, write
+    # t.provisioned_throughput({
+    #   read_capacity_units: 1,
+    #   write_capacity_units: 1
+    # }
+    def provisioned_throughput(*args)
+      if args.size == 0 # reader method
+        @provisioned_throughput # early return
+      elsif args.first.is_a?(Hash)
+        # @provisioned_throughput_set_called useful for update_table
+        # only provide a provisioned_throughput settings if explicitly called for update_table
+        @provisioned_throughput_set_called = true
+        # Case:
+        # provisioned_throughput(
+        #   read_capacity_units: 1,
+        #   write_capacity_units: 1
+        # )
+        @provisioned_throughput = arg.first # set directly
+      else  # assume parameter is an Integer or [Integer, Integer]
+        # Case: provisioned_throughput(1)
+        # Case: provisioned_throughput(1, 1)
+        read_capacity_units, write_capacity_units = args
+        @provisioned_throughput = {
+          read_capacity_units: read_capacity_units,
+          write_capacity_units: write_capacity_units || read_capacity_units,
+        }
+      end
+    end
+  end
+end

data/lib/dynomite/migration/dsl.rb

@@ -1,35 +1,27 @@
 class Dynomite::Migration
   class Dsl
-    ATTRIBUTES = %i[
-      key_schema
-      attribute_definitions
-      table_name
-      billing_mode
-    ].freeze
-
-    autoload :Common, "dynomite/migration/common"
-    autoload :BaseSecondaryIndex, "dynomite/migration/dsl/base_secondary_index"
-    autoload :LocalSecondaryIndex, "dynomite/migration/dsl/local_secondary_index"
-    autoload :GlobalSecondaryIndex, "dynomite/migration/dsl/global_secondary_index"
-
-    include Dynomite::DbConfig
-    include Common
-
-    attr_accessor(*ATTRIBUTES)
-
+    extend Accessor
+    include Dynomite::Client
+    include ProvisionedThroughput
+    include PrimaryKey
+    include Index
+    include Helpers
+
+    attr_accessor :attribute_definitions,
+                  :key_schema,
+                  :table_name
     def initialize(method_name, table_name, &block)
       @method_name = method_name
       @table_name = table_name
       @block = block
 
-      # Dsl fills in attributes in as methods are called within the block.
+      # Dsl fills in atttributes in as methods are called within the block
+      # when parition_key and sort_key are called.
       # Attributes for both create_table and updated_table:
-      @billing_mode = 'PROVISIONED'
       @attribute_definitions = []
-      @provisioned_throughput = {
-        read_capacity_units: 5,
-        write_capacity_units: 5
-      }
+      # dont set billing_mode for update_table. otherwise creating an index can set billing_mode to PAY_PER_REQUEST without user knowing
+      @billing_mode = "PAY_PER_REQUEST" if @method_name == :create_table
+      @provisioned_throughput = nil
 
       # Attributes for create_table only:
       @key_schema = []
@@ -39,35 +31,10 @@ class Dynomite::Migration
       @lsi_indexes = []
     end
 
-    # t.billing_mode(:pay_per_request)
-    # t.billing_mode(:provisioned) # default value
-    def billing_mode(mode = nil)
-      return @billing_mode if mode.nil?
-
-      @billing_mode = mode.to_s.upcase
+    def namespaced_table_name
+      table_name_with_namespace(table_name)
     end
 
-    # t.gsi(:create) do |i|
-    #   i.partition_key "category:string"
-    #   i.sort_key "created_at:string" # optional
-    # end
-    def gsi(action=:create, index_name=nil, &block)
-      gsi_index = GlobalSecondaryIndex.new(action, index_name, &block)
-      @gsi_indexes << gsi_index # store @gsi_index for the parent Dsl to use
-    end
-    alias_method :global_secondary_index, :gsi
-
-    # t.lsi(:create) do |i|
-    #   i.partition_key "category:string"
-    #   i.sort_key "created_at:string" # optional
-    # end
-    def lsi(action=:create, index_name=nil, &block)
-      # dont need action create but have it to keep the lsi and gsi method consistent
-      lsi_index = LocalSecondaryIndex.new(index_name, &block)
-      @lsi_indexes << lsi_index # store @lsi_index for the parent Dsl to use
-    end
-    alias_method :local_secondary_index, :lsi
-
     def evaluate
       return if @evaluated
       @block.call(self) if @block
@@ -79,136 +46,116 @@ class Dynomite::Migration
       # executor
     def params
       evaluate # lazy evaluation: wait until as long as possible before evaluating code block
-
-      # Not using send because think its clearer in this case
-      case @method_name
-      when :create_table
-        params_create_table
-      when :update_table
-        params_update_table
-      end
+      send "params_#{@method_name}" # IE: params_create_table, params_update_table, params_delete_table
     end
 
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#create_table-instance_method
     def params_create_table
-      merge_lsi_attribute_definitions!
-      merge_gsi_attribute_definitions!
-
       params = {
+        billing_mode: @billing_mode.to_s.upcase,
         table_name: namespaced_table_name,
         key_schema: @key_schema,
-        attribute_definitions: @attribute_definitions,
-        billing_mode: @billing_mode
+        provisioned_throughput: @provisioned_throughput,
+        tags: @tags, # only for create_table
       }
+      params.reject! { |k,v| v.blank? }
 
-      params[:provisioned_throughput] = @provisioned_throughput if @billing_mode == 'PROVISIONED'
       params[:local_secondary_indexes] = lsi_secondary_index_creates unless @lsi_indexes.empty?
       params[:global_secondary_indexes] = gsi_secondary_index_creates unless @gsi_indexes.empty?
+      params[:attribute_definitions] = attribute_definitions
+      params.merge!(common_params)
       params
     end
 
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#update_table-instance_method
+    dsl_accessor :replica_updates
     def params_update_table
-      merge_gsi_attribute_definitions!
-
       params = {
+        billing_mode: @billing_mode.to_s.upcase,
         table_name: namespaced_table_name,
-        attribute_definitions: @attribute_definitions,
-        billing_mode: @billing_mode
         # update table take values only some values for the "parent" table
         # no key_schema, update_table does not handle key_schema for the "parent" table
+        replica_updates: @replica_updates, # only for update_table
       }
-      # only set "parent" table provisioned_throughput if user actually invoked
-      # it in the dsl
-      if @provisioned_throughput_set_called && @billing_mode == 'PROVISIONED'
-        params[:provisioned_throughput] = @provisioned_throughput
-      end
-      params[:global_secondary_index_updates] = global_secondary_index_updates
+      params.reject! { |k,v| v.blank? }
+
+      # only set "parent" table provisioned_throughput if user actually invoked it in the dsl
+      params[:provisioned_throughput] = @provisioned_throughput if @provisioned_throughput_set_called
+      params[:global_secondary_index_updates] = global_secondary_index_updates unless @gsi_indexes.empty?
+      params[:attribute_definitions] = attribute_definitions
       params
     end
 
-    # Goes thorugh all the lsi_indexes that have been built up in memory.
-    # Find the lsi object that creates an index and then grab the
-    # attribute_definitions from it.
-    def merge_lsi_attribute_definitions!
-      lsi = @lsi_indexes.first # DynamoDB only supports adding one index at a time anyway. The reason @lsi_indexes is an Array is because we're sharing the same class code for LSI and GSI
-      if lsi
-        lsi.evaluate # force early evaluate since we need the params to
-          # add: gsi_attribute_definitions + lsi_attrs
-        lsi_attrs = lsi.attribute_definitions
-      end
-      all_attrs = if lsi_attrs
-                    @attribute_definitions + lsi_attrs
-                  else
-                    @attribute_definitions
-                  end
-      @attribute_definitions = all_attrs.uniq
+    def params_delete_table
+      { table_name: namespaced_table_name }
     end
 
-    # maps each lsi to the hash structure expected by dynamodb update_table
-    # under the global_secondary_index_updates key:
-    #
-    #   { create: {...} }
-    #   { update: {...} }
-    #   { delete: {...} }
-    def lsi_secondary_index_creates
-      @lsi_indexes.map do |lsi|
-        lsi.params
+    def tags(values=nil)
+      if values.nil?
+        @tags
+      else
+        if values.is_a?(Hash)
+          @tags = values.map do |key,value|
+            {key: key.to_s, value: value.to_s}
+          end
+        else
+          @tags = values
+        end
       end
     end
 
-    # maps each lsi to the hash structure expected by dynamodb update_table
-    # under the global_secondary_index_updates key:
-    #
-    #   { create: {...} }
-    #   { update: {...} }
-    #   { delete: {...} }
-    def gsi_secondary_index_creates
-      @gsi_indexes.map do |gsi|
-        gsi.params
+    # @attribute_definitions only hold the partition_key and sort_key attributes
+    # We'll also added the attributes from: 1. lsi index 2. gsi indexes 3. existing attributes for update table
+    def attribute_definitions
+      # Goes through all the lsi_indexes that have been built up in memory.
+      # Find the lsi object that creates an index and then grab the attribute_definitions from it.
+      # All lsi indexes are create. There is no update or delete for lsi indexes.
+      @lsi_indexes.each do |lsi|
+        lsi.attribute_definitions.each do |definition|
+          @attribute_definitions << definition unless @attribute_definitions.include?(definition)
+        end
       end
-    end
 
-    # Goes thorugh all the gsi_indexes that have been built up in memory.
-    # Find the gsi object that creates an index and then grab the
-    # attribute_definitions from it.
-    def merge_gsi_attribute_definitions!
-      gsi_attrs = []
-
-      gsi = @gsi_indexes.find { |i| i.action == :create }
-      if gsi
-        gsi.evaluate # force early evaluate since we need the params to
-          # add: gsi_attribute_definitions + gsi_attrs
-        gsi_attrs = gsi.attribute_definitions
+      # Goes through all the gsi_indexes that have been built up in memory.
+      # Find the gsi object that creates an index and then grab the attribute_definitions from it.
+      @gsi_indexes.select { |i| i.action == :create }.each do |gsi|
+        gsi.attribute_definitions.each do |definition|
+          @attribute_definitions << definition unless @attribute_definitions.include?(definition)
+        end
       end
 
       # Merge existing attributes for update table
-      all_gsi_attrs = @method_name == :update_table ?
-        gsi_attribute_definitions + gsi_attrs :
-        gsi_attrs
+      if @method_name == :update_table
+        existing_attributes = desc_table(namespaced_table_name).attribute_definitions
+        existing_attributes.each do |attribute_definition|
+          definition = attribute_definition.to_h
+          @attribute_definitions << definition unless @attribute_definitions.include?(definition)
+        end
+      end
 
-      all_attrs = (@attribute_definitions + all_gsi_attrs).uniq
-      @attribute_definitions = all_attrs
+      @attribute_definitions
     end
 
-    # maps each gsi to the hash structure expected by dynamodb update_table
-    # under the global_secondary_index_updates key:
-    #
-    #   { create: {...} }
-    #   { update: {...} }
-    #   { delete: {...} }
-    def global_secondary_index_updates
-      @gsi_indexes.map do |gsi|
-        { gsi.action => gsi.params }
-      end
+    dsl_accessor :stream_specification, :sse_specification
+    # common to create_table and update_table
+    def common_params
+      params = {
+        stream_specification: @stream_specification,
+        sse_specification: @sse_specification,
+        deletion_protection_enabled: deletion_protection_enabled_value,
+      }
+      params.reject! { |k,v| v.blank? }
     end
 
-    # >> resp = Post.db.describe_table(table_name: "demo-dev-posts")
-    # >> resp.table.attribute_definitions.map(&:to_h)
-    # => [{:attribute_name=>"id", :attribute_type=>"S"}]
-    def gsi_attribute_definitions
-      return @gsi_attribute_definitions if @gsi_attribute_definitions
+    def deletion_protection_enabled(value=true)
+      @deletion_protection_enabled = value
+    end
+    alias deletion_protection deletion_protection_enabled
 
-      resp = db.describe_table(table_name: namespaced_table_name)
-      @gsi_attribute_definitions = resp.table.attribute_definitions.map(&:to_h)
+    # To avoid name conflict with DSL t.deletion_protection_enabled
+    def deletion_protection_enabled_value
+      default = Dynomite.config.migration.deletion_protection_enabled
+      @deletion_protection_enabled.nil? ? default : @deletion_protection_enabled
     end
   end
 end

data/lib/dynomite/migration/file_info.rb

@@ -0,0 +1,28 @@
+class Dynomite::Migration
+  class FileInfo
+    attr_reader :path
+    def initialize(path)
+      @path = path
+    end
+
+    def pretty_path
+      @path.sub(/^\.\//,'') # remove leading ./
+    end
+
+    def migration_class
+      filename = File.basename(@path, '.rb')
+      filename = filename.sub(/\d+[-_]/, '') # strip leading timestsamp
+      filename.camelize.constantize
+    end
+
+    def version
+      filename = File.basename(@path, '.rb')
+      md = filename.match(/(\d+)[-_]/)
+      md[1].to_i
+    end
+
+    def self.all_files
+      Dir.glob("#{Dynomite.root}/dynamodb/migrate/*").to_a.sort
+    end
+  end
+end