dynomite 1.2.7 → 2.0.1

data/lib/dynomite/migration/dsl/primary_key.rb

@@ -0,0 +1,62 @@
+# Common methods to the *SecondaryIndex classes that handle gsi and lsi methods
+# as well a the Dsl class that handles create_table and update_table methods.
+class Dynomite::Migration::Dsl
+  module PrimaryKey
+    include Dynomite::Types
+
+    # http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Types/KeySchemaElement.html
+    # partition_key is required
+    attr_reader :partition_key_identifier
+    def partition_key(identifier)
+      identifier = identifier.to_s
+      @partition_key_identifier = identifier # for later use. useful for conventional_index_name
+      adjust_schema_and_attributes(identifier, "HASH")
+    end
+
+    # sort_key is optional
+    attr_reader :sort_key_identifier
+    def sort_key(identifier)
+      identifier = identifier.to_s
+      @sort_key_identifier = identifier # for later use. useful for conventional_index_name
+      adjust_schema_and_attributes(identifier, "RANGE")
+    end
+
+    # Use class variable to share across all instances of GlobalSecondaryIndex
+    # Note: Even though dynomite merges attribute defintions correctly for
+    # multiple indexes, DynamoDB does not allow creating multiple indexes
+    # at the same time. Example error:
+    #   Unable to update table: Subscriber limit exceeded: Only 1 online index can be created or deleted simultaneously per table
+    # Leaving the logic in place in case AWS changes this in the future.
+
+    # Parameters:
+    #   identifier: "id:string" or "id"
+    #   key_type: "HASH" OR "RANGE"
+    #
+    # Adjusts the parameters for create_table to add the
+    # partition_key and sort_key
+    def adjust_schema_and_attributes(identifier, key_type)
+      name, attribute_type = identifier.split(':')
+      attribute_type ||= "string" # default to string
+
+      partition_key = {
+        attribute_name: name,
+        key_type: key_type.upcase
+      }
+
+      if partition_key[:key_type] == "RANGE"
+        @key_schema << partition_key unless @key_schema.include?(partition_key)
+      else # HASH - add to beginning
+        @key_schema.unshift(partition_key) unless @key_schema.include?(partition_key)
+      end
+
+      attribute_definition = {
+        attribute_name: name,
+        attribute_type: type_map(attribute_type)
+      }
+      unless @attribute_definitions.include?(attribute_definition)
+        @attribute_definitions << attribute_definition
+      end
+      @attribute_definitions
+    end
+  end
+end

data/lib/dynomite/migration/dsl/provisioned_throughput.rb

@@ -0,0 +1,38 @@
+class Dynomite::Migration::Dsl
+  module ProvisionedThroughput
+    def billing_mode(v)
+      @billing_mode = v
+      @provisioned_throughput = nil if @billing_mode.to_s.upcase == "PAY_PER_REQUEST"
+    end
+
+    # t.provisioned_throughput(1) # both
+    # t.provisioned_throughput(1,1) # read, write
+    # t.provisioned_throughput({
+    #   read_capacity_units: 1,
+    #   write_capacity_units: 1
+    # }
+    def provisioned_throughput(*args)
+      if args.size == 0 # reader method
+        @provisioned_throughput # early return
+      elsif args.first.is_a?(Hash)
+        # @provisioned_throughput_set_called useful for update_table
+        # only provide a provisioned_throughput settings if explicitly called for update_table
+        @provisioned_throughput_set_called = true
+        # Case:
+        # provisioned_throughput(
+        #   read_capacity_units: 1,
+        #   write_capacity_units: 1
+        # )
+        @provisioned_throughput = arg.first # set directly
+      else  # assume parameter is an Integer or [Integer, Integer]
+        # Case: provisioned_throughput(1)
+        # Case: provisioned_throughput(1, 1)
+        read_capacity_units, write_capacity_units = args
+        @provisioned_throughput = {
+          read_capacity_units: read_capacity_units,
+          write_capacity_units: write_capacity_units || read_capacity_units,
+        }
+      end
+    end
+  end
+end

data/lib/dynomite/migration/dsl.rb

@@ -1,35 +1,27 @@
 class Dynomite::Migration
   class Dsl
-    ATTRIBUTES = %i[
-      key_schema
-      attribute_definitions
-      table_name
-      billing_mode
-    ].freeze
-
-    autoload :Common, "dynomite/migration/common"
-    autoload :BaseSecondaryIndex, "dynomite/migration/dsl/base_secondary_index"
-    autoload :LocalSecondaryIndex, "dynomite/migration/dsl/local_secondary_index"
-    autoload :GlobalSecondaryIndex, "dynomite/migration/dsl/global_secondary_index"
-
-    include Dynomite::DbConfig
-    include Common
-
-    attr_accessor(*ATTRIBUTES)
-
+    extend Accessor
+    include Dynomite::Client
+    include ProvisionedThroughput
+    include PrimaryKey
+    include Index
+    include Helpers
+
+    attr_accessor :attribute_definitions,
+                  :key_schema,
+                  :table_name
     def initialize(method_name, table_name, &block)
       @method_name = method_name
       @table_name = table_name
       @block = block
 
-      # Dsl fills in attributes in as methods are called within the block.
+      # Dsl fills in atttributes in as methods are called within the block
+      # when parition_key and sort_key are called.
       # Attributes for both create_table and updated_table:
-      @billing_mode = 'PROVISIONED'
       @attribute_definitions = []
-      @provisioned_throughput = {
-        read_capacity_units: 5,
-        write_capacity_units: 5
-      }
+      # dont set billing_mode for update_table. otherwise creating an index can set billing_mode to PAY_PER_REQUEST without user knowing
+      @billing_mode = "PAY_PER_REQUEST" if @method_name == :create_table
+      @provisioned_throughput = nil
 
       # Attributes for create_table only:
       @key_schema = []
@@ -39,35 +31,10 @@ class Dynomite::Migration
       @lsi_indexes = []
     end
 
-    # t.billing_mode(:pay_per_request)
-    # t.billing_mode(:provisioned) # default value
-    def billing_mode(mode = nil)
-      return @billing_mode if mode.nil?
-
-      @billing_mode = mode.to_s.upcase
+    def namespaced_table_name
+      table_name_with_namespace(table_name)
     end
 
-    # t.gsi(:create) do |i|
-    #   i.partition_key "category:string"
-    #   i.sort_key "created_at:string" # optional
-    # end
-    def gsi(action=:create, index_name=nil, &block)
-      gsi_index = GlobalSecondaryIndex.new(action, index_name, &block)
-      @gsi_indexes << gsi_index # store @gsi_index for the parent Dsl to use
-    end
-    alias_method :global_secondary_index, :gsi
-
-    # t.lsi(:create) do |i|
-    #   i.partition_key "category:string"
-    #   i.sort_key "created_at:string" # optional
-    # end
-    def lsi(action=:create, index_name=nil, &block)
-      # dont need action create but have it to keep the lsi and gsi method consistent
-      lsi_index = LocalSecondaryIndex.new(index_name, &block)
-      @lsi_indexes << lsi_index # store @lsi_index for the parent Dsl to use
-    end
-    alias_method :local_secondary_index, :lsi
-
     def evaluate
       return if @evaluated
       @block.call(self) if @block
@@ -79,136 +46,116 @@ class Dynomite::Migration
       # executor
     def params
       evaluate # lazy evaluation: wait until as long as possible before evaluating code block
-
-      # Not using send because think its clearer in this case
-      case @method_name
-      when :create_table
-        params_create_table
-      when :update_table
-        params_update_table
-      end
+      send "params_#{@method_name}" # IE: params_create_table, params_update_table, params_delete_table
     end
 
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#create_table-instance_method
     def params_create_table
-      merge_lsi_attribute_definitions!
-      merge_gsi_attribute_definitions!
-
       params = {
+        billing_mode: @billing_mode.to_s.upcase,
         table_name: namespaced_table_name,
         key_schema: @key_schema,
-        attribute_definitions: @attribute_definitions,
-        billing_mode: @billing_mode
+        provisioned_throughput: @provisioned_throughput,
+        tags: @tags, # only for create_table
       }
+      params.reject! { |k,v| v.blank? }
 
-      params[:provisioned_throughput] = @provisioned_throughput if @billing_mode == 'PROVISIONED'
       params[:local_secondary_indexes] = lsi_secondary_index_creates unless @lsi_indexes.empty?
       params[:global_secondary_indexes] = gsi_secondary_index_creates unless @gsi_indexes.empty?
+      params[:attribute_definitions] = attribute_definitions
+      params.merge!(common_params)
       params
     end
 
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#update_table-instance_method
+    dsl_accessor :replica_updates
     def params_update_table
-      merge_gsi_attribute_definitions!
-
       params = {
+        billing_mode: @billing_mode.to_s.upcase,
         table_name: namespaced_table_name,
-        attribute_definitions: @attribute_definitions,
-        billing_mode: @billing_mode
         # update table take values only some values for the "parent" table
         # no key_schema, update_table does not handle key_schema for the "parent" table
+        replica_updates: @replica_updates, # only for update_table
       }
-      # only set "parent" table provisioned_throughput if user actually invoked
-      # it in the dsl
-      if @provisioned_throughput_set_called && @billing_mode == 'PROVISIONED'
-        params[:provisioned_throughput] = @provisioned_throughput
-      end
-      params[:global_secondary_index_updates] = global_secondary_index_updates
+      params.reject! { |k,v| v.blank? }
+
+      # only set "parent" table provisioned_throughput if user actually invoked it in the dsl
+      params[:provisioned_throughput] = @provisioned_throughput if @provisioned_throughput_set_called
+      params[:global_secondary_index_updates] = global_secondary_index_updates unless @gsi_indexes.empty?
+      params[:attribute_definitions] = attribute_definitions
       params
     end
 
-    # Goes thorugh all the lsi_indexes that have been built up in memory.
-    # Find the lsi object that creates an index and then grab the
-    # attribute_definitions from it.
-    def merge_lsi_attribute_definitions!
-      lsi = @lsi_indexes.first # DynamoDB only supports adding one index at a time anyway. The reason @lsi_indexes is an Array is because we're sharing the same class code for LSI and GSI
-      if lsi
-        lsi.evaluate # force early evaluate since we need the params to
-          # add: gsi_attribute_definitions + lsi_attrs
-        lsi_attrs = lsi.attribute_definitions
-      end
-      all_attrs = if lsi_attrs
-                    @attribute_definitions + lsi_attrs
-                  else
-                    @attribute_definitions
-                  end
-      @attribute_definitions = all_attrs.uniq
+    def params_delete_table
+      { table_name: namespaced_table_name }
     end
 
-    # maps each lsi to the hash structure expected by dynamodb update_table
-    # under the global_secondary_index_updates key:
-    #
-    #   { create: {...} }
-    #   { update: {...} }
-    #   { delete: {...} }
-    def lsi_secondary_index_creates
-      @lsi_indexes.map do |lsi|
-        lsi.params
+    def tags(values=nil)
+      if values.nil?
+        @tags
+      else
+        if values.is_a?(Hash)
+          @tags = values.map do |key,value|
+            {key: key.to_s, value: value.to_s}
+          end
+        else
+          @tags = values
+        end
       end
     end
 
-    # maps each lsi to the hash structure expected by dynamodb update_table
-    # under the global_secondary_index_updates key:
-    #
-    #   { create: {...} }
-    #   { update: {...} }
-    #   { delete: {...} }
-    def gsi_secondary_index_creates
-      @gsi_indexes.map do |gsi|
-        gsi.params
+    # @attribute_definitions only hold the partition_key and sort_key attributes
+    # We'll also added the attributes from: 1. lsi index 2. gsi indexes 3. existing attributes for update table
+    def attribute_definitions
+      # Goes through all the lsi_indexes that have been built up in memory.
+      # Find the lsi object that creates an index and then grab the attribute_definitions from it.
+      # All lsi indexes are create. There is no update or delete for lsi indexes.
+      @lsi_indexes.each do |lsi|
+        lsi.attribute_definitions.each do |definition|
+          @attribute_definitions << definition unless @attribute_definitions.include?(definition)
+        end
       end
-    end
 
-    # Goes thorugh all the gsi_indexes that have been built up in memory.
-    # Find the gsi object that creates an index and then grab the
-    # attribute_definitions from it.
-    def merge_gsi_attribute_definitions!
-      gsi_attrs = []
-
-      gsi = @gsi_indexes.find { |i| i.action == :create }
-      if gsi
-        gsi.evaluate # force early evaluate since we need the params to
-          # add: gsi_attribute_definitions + gsi_attrs
-        gsi_attrs = gsi.attribute_definitions
+      # Goes through all the gsi_indexes that have been built up in memory.
+      # Find the gsi object that creates an index and then grab the attribute_definitions from it.
+      @gsi_indexes.select { |i| i.action == :create }.each do |gsi|
+        gsi.attribute_definitions.each do |definition|
+          @attribute_definitions << definition unless @attribute_definitions.include?(definition)
+        end
       end
 
       # Merge existing attributes for update table
-      all_gsi_attrs = @method_name == :update_table ?
-        gsi_attribute_definitions + gsi_attrs :
-        gsi_attrs
+      if @method_name == :update_table
+        existing_attributes = desc_table(namespaced_table_name).attribute_definitions
+        existing_attributes.each do |attribute_definition|
+          definition = attribute_definition.to_h
+          @attribute_definitions << definition unless @attribute_definitions.include?(definition)
+        end
+      end
 
-      all_attrs = (@attribute_definitions + all_gsi_attrs).uniq
-      @attribute_definitions = all_attrs
+      @attribute_definitions
     end
 
-    # maps each gsi to the hash structure expected by dynamodb update_table
-    # under the global_secondary_index_updates key:
-    #
-    #   { create: {...} }
-    #   { update: {...} }
-    #   { delete: {...} }
-    def global_secondary_index_updates
-      @gsi_indexes.map do |gsi|
-        { gsi.action => gsi.params }
-      end
+    dsl_accessor :stream_specification, :sse_specification
+    # common to create_table and update_table
+    def common_params
+      params = {
+        stream_specification: @stream_specification,
+        sse_specification: @sse_specification,
+        deletion_protection_enabled: deletion_protection_enabled_value,
+      }
+      params.reject! { |k,v| v.blank? }
     end
 
-    # >> resp = Post.db.describe_table(table_name: "demo-dev-posts")
-    # >> resp.table.attribute_definitions.map(&:to_h)
-    # => [{:attribute_name=>"id", :attribute_type=>"S"}]
-    def gsi_attribute_definitions
-      return @gsi_attribute_definitions if @gsi_attribute_definitions
+    def deletion_protection_enabled(value=true)
+      @deletion_protection_enabled = value
+    end
+    alias deletion_protection deletion_protection_enabled
 
-      resp = db.describe_table(table_name: namespaced_table_name)
-      @gsi_attribute_definitions = resp.table.attribute_definitions.map(&:to_h)
+    # To avoid name conflict with DSL t.deletion_protection_enabled
+    def deletion_protection_enabled_value
+      default = Dynomite.config.migration.deletion_protection_enabled
+      @deletion_protection_enabled.nil? ? default : @deletion_protection_enabled
     end
   end
 end

data/lib/dynomite/migration/file_info.rb

@@ -0,0 +1,28 @@
+class Dynomite::Migration
+  class FileInfo
+    attr_reader :path
+    def initialize(path)
+      @path = path
+    end
+
+    def pretty_path
+      @path.sub(/^\.\//,'') # remove leading ./
+    end
+
+    def migration_class
+      filename = File.basename(@path, '.rb')
+      filename = filename.sub(/\d+[-_]/, '') # strip leading timestsamp
+      filename.camelize.constantize
+    end
+
+    def version
+      filename = File.basename(@path, '.rb')
+      md = filename.match(/(\d+)[-_]/)
+      md[1].to_i
+    end
+
+    def self.all_files
+      Dir.glob("#{Dynomite.root}/dynamodb/migrate/*").to_a.sort
+    end
+  end
+end

data/lib/dynomite/migration/generator.rb

@@ -3,8 +3,6 @@ require "active_support/core_ext/string"
 class Dynomite::Migration
   # jets dynamodb:generate posts --partition-key id:string
   class Generator
-    include Dynomite::DbConfig
-
     attr_reader :migration_name, :table_name
     def initialize(migration_name, options)
       @migration_name = migration_name
@@ -12,21 +10,28 @@ class Dynomite::Migration
     end
 
     def generate
-      puts "Generating migration for #{@table_name}" unless @options[:quiet]
-      return if @options[:noop]
+      puts "Generating migration" unless @options[:quiet]
+      return if ENV['NOOP']
       create_migration
     end
 
     def create_migration
       FileUtils.mkdir_p(File.dirname(migration_path))
       IO.write(migration_path, migration_code)
-      puts "Migration file created: #{migration_path}. \nTo run:"
-      puts "  jets dynamodb:migrate #{migration_path}"
+      pretty_migration_path = migration_path.sub(/^\.\//,'') # remove leading ./
+      puts "Migration file created: #{pretty_migration_path}\nTo run:\n\n"
+      command = File.basename($0)
+      if command == "jets"
+        puts "    #{command} dynamodb:migrate"
+      else
+        puts "    #{command} migrate"
+      end
+      puts
     end
 
     def migration_code
-      path = File.expand_path("../templates/#{table_action}.rb", __FILE__)
-      result = Dynomite::Erb.result(path,
+      path = File.expand_path("../templates/#{action}.rb", __FILE__)
+      Dynomite::Erb.result(path,
         migration_class_name: migration_class_name,
         table_name: table_name,
         partition_key: @options[:partition_key],
@@ -35,30 +40,39 @@ class Dynomite::Migration
       )
     end
 
-    def table_action
-      @options[:table_action] || conventional_table_action
+    def action
+      # optoins[:table_action] is old and deprecated
+      action = @options[:action] || @options[:table_action] || conventional_action
+      case action
+      when /create/
+        "create_table"
+      when /delete/
+        "delete_table"
+      else
+        "update_table" # default and fallback
+      end
     end
 
-    def conventional_table_action
-      @migration_name.include?("update") ? "update_table" : "create_table"
+    def conventional_action
+      @migration_name.split("_").first
     end
 
     def table_name
-      @options[:table_name] || conventional_table_name
+      @options[:table_name] || conventional_table_name || "TABLE_NAME"
     end
 
     # create_posts => posts
     # update_posts => posts
     def conventional_table_name
-      @migration_name.sub(/^(create|update)_/, '')
+      @migration_name.sub(/^(create|update|delete)_/, '')
     end
 
     def migration_class_name
-      "#{@migration_name}_migration".classify # doesnt include timestamp
+      "#{@migration_name}".camelize # doesnt include timestamp
     end
 
     def migration_path
-      "#{Dynomite.app_root}dynamodb/migrate/#{timestamp}-#{@migration_name}_migration.rb"
+      "#{Dynomite.root}/dynamodb/migrate/#{timestamp}-#{@migration_name}.rb"
     end
 
     def timestamp

data/lib/dynomite/migration/helpers.rb

@@ -0,0 +1,7 @@
+class Dynomite::Migration
+  module Helpers
+    def table_name_with_namespace(table_name)
+      [Dynomite.config.namespace, table_name].reject {|s| s.nil? || s.empty?}.join(Dynomite.config.namespace_separator)
+    end
+  end
+end

data/lib/dynomite/migration/internal/migrate/create_schema_migrations.rb

@@ -0,0 +1,17 @@
+class CreateSchemaMigrations < Dynomite::Migration
+  include Dynomite::Client
+
+  def up
+    create_table :schema_migrations do |t|
+      t.partition_key "version:number" # required
+      t.billing_mode "PAY_PER_REQUEST"
+    end
+  end
+
+  def table_exist?(full_table_name)
+    client.describe_table(table_name: full_table_name)
+    true
+  rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+    false
+  end
+end

data/lib/dynomite/migration/internal/models/schema_migration.rb

@@ -0,0 +1,6 @@
+class SchemaMigration < Dynomite::Item
+  disable_id!
+  field :version, type: :integer
+  field :time_took, type: :integer
+  fields :status, :path, :error_message
+end