dyna 0.1.9 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: bf7e0af4635d19cc04464734c752ec8f68032ea3
-  data.tar.gz: 68e76221165d0268c622001e07190ca753894c81
+SHA256:
+  metadata.gz: e4c57e85bcc9f5bce4b084da7b9c6c7338529d1b83512b1549bc31f47d48bc1e
+  data.tar.gz: 5a14ba531ff8a96bb56ccd09af7db82d8c9e6a3024f3051be520203a890d382e
 SHA512:
-  metadata.gz: 9940c92ee88b999d0a9a5a8ba7f745b4d9afd900773fdd7b512c85d289318dbbd81877c4217e569a7ee2b697dabcb0c61d9ac2be45c67cc3ff464dc01def0914
-  data.tar.gz: 0a2aee89b5f14f22171dfd4a43b9235d9143af4b39b44e66023b0137b53037874b57d89975d70ec9f7864c74ed440f17d5108dbc706c023d5e02b2924a055e0b
+  metadata.gz: cf9547e5373b6b690623732de1c70a2efce0128863f53a92a1cbffcc7539ca5294299a993ecf587af339a1fce3fd7ef3f549e5f4313e3b3283003ccb9a54f28c
+  data.tar.gz: cbefd166bad5e80202ef3d0f9e5f73da90642fbb0dbce0e6a1c406e85143cae539fc0f7464c0240c88d6c31a21c1f1f6c3c6d4d2d654e069342243c14c7a0d67

data/dyna.gemspec

@@ -20,7 +20,8 @@ Gem::Specification.new do |spec|
   spec.bindir        = "bin"
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.add_dependency 'aws-sdk', '~> 2'
+  spec.add_dependency 'aws-sdk-dynamodb', '~> 1.18'
+  spec.add_dependency 'aws-sdk-applicationautoscaling', '~> 1.16'
   spec.add_dependency 'term-ansicolor', '~> 1.4'
   spec.add_dependency 'diffy', '~> 3.1'
   spec.add_dependency 'hashie', '~> 3.4'

data/lib/dyna.rb

@@ -6,7 +6,8 @@ require 'hashie'
 require 'singleton'
 require 'pp'
 
-require 'aws-sdk'
+require 'aws-sdk-applicationautoscaling'
+require 'aws-sdk-dynamodb'
 
 require 'dyna/version'
 require 'dyna/logger'

data/lib/dyna/client.rb

@@ -15,6 +15,8 @@ module Dyna
         )
       else
         @options.ddb = Aws::DynamoDB::Client.new
+        @options.aas = Aws::ApplicationAutoScaling::Client.new
+        Exporter.aas(@options.aas)
       end
     end
 

data/lib/dyna/dsl/converter.rb

@@ -35,41 +35,42 @@ end
         stream_specification = ''
         if table[:local_secondary_indexes]
           local_secondary_indexes_tmpl = <<-EOS.chomp
-<% table[:local_secondary_indexes].each do |index| %>
+<% table[:local_secondary_indexes].each do |index| -%>
     local_secondary_index <%= index[:index_name].inspect %> do
       key_schema hash: <%= index[:key_schema][0][:attribute_name].inspect %>, range: <%= index[:key_schema][1][:attribute_name].inspect %><% if index[:projection] %>
       projection projection_type: <%= index[:projection][:projection_type].inspect %><% end %>
     end
-<% end %>
+<%- end %>
 EOS
-          local_secondary_indexes = ERB.new(local_secondary_indexes_tmpl).result(binding)
+          local_secondary_indexes = ERB.new(local_secondary_indexes_tmpl, nil, '-').result(binding)
         end
 
         if table[:global_secondary_indexes]
           global_secondary_indexes_tmpl = <<-EOS.chomp
-<% table[:global_secondary_indexes].each do |index| %>
+<% table[:global_secondary_indexes].each do |index| -%>
     global_secondary_index <%= index[:index_name].inspect %> do
       key_schema hash: <%= index[:key_schema][0][:attribute_name].inspect %><% if index[:key_schema].size == 2 %>, range: <%= index[:key_schema][1][:attribute_name].inspect %><% end %><% if index[:projection] %>
       projection projection_type: <%= index[:projection][:projection_type].inspect %><% end %>
       provisioned_throughput read_capacity_units: <%= index[:provisioned_throughput][:read_capacity_units] %>, write_capacity_units: <%= index[:provisioned_throughput][:read_capacity_units] %>
     end
-<% end %>
+<%- end %>
 EOS
-          global_secondary_indexes = ERB.new(global_secondary_indexes_tmpl).result(binding)
+          global_secondary_indexes = ERB.new(global_secondary_indexes_tmpl, nil, '-').result(binding)
         end
 
         attribute_definitions_tmpl = <<-EOS.chomp
-<% table[:attribute_definitions].each do |attr| %>
+<% table[:attribute_definitions].each do |attr| -%>
+
     attribute_definition(
       attribute_name: <%= attr[:attribute_name].inspect %>,
       attribute_type: <%= attr[:attribute_type].inspect %>,
     )
-<% end %>
+<%- end %>
 EOS
-        attribute_definitions = ERB.new(attribute_definitions_tmpl).result(binding)
+        attribute_definitions = ERB.new(attribute_definitions_tmpl, nil, '-').result(binding)
 
         if table[:stream_specification]
-          stream_specification_tmpl = <<-EOS.chomp
+          stream_specification_tmpl = <<-EOS
 
     stream_specification(
       stream_enabled: <%= table[:stream_specification][:stream_enabled] %>,
@@ -79,6 +80,35 @@ EOS
           stream_specification = ERB.new(stream_specification_tmpl).result(binding)
         end
 
+        if table[:scalable_targets]
+          scalable_targets_tmpl = <<-EOS.chomp
+<% table[:scalable_targets].each do |target| -%>
+
+    scalable_target(
+      scalable_dimension: <%= target[:scalable_dimension].inspect %>,
+      min_capacity: <%= target[:min_capacity] %>,
+      max_capacity: <%= target[:max_capacity] %>,
+    )
+<%- end %>
+EOS
+          scalable_targets = ERB.new(scalable_targets_tmpl, nil, '-').result(binding)
+        end
+
+        if table[:scaling_policies]
+          scaling_policies_tmpl = <<-EOS.chomp
+<% table[:scaling_policies].each do |policy| -%>
+
+    scaling_policy(
+      scalable_dimension: <%= policy[:scalable_dimension].inspect %>,
+      target_tracking_scaling_policy_configuration: {
+        target_value: <%= policy[:target_tracking_scaling_policy_configuration][:target_value] %>,
+      },
+    )
+<%- end %>
+EOS
+          scaling_policies = ERB.new(scaling_policies_tmpl, nil, '-').result(binding)
+        end
+
         <<-EOS
   table "#{name}" do
     key_schema(
@@ -90,7 +120,9 @@ EOS
       read_capacity_units: #{table[:provisioned_throughput][:read_capacity_units]},
       write_capacity_units: #{table[:provisioned_throughput][:write_capacity_units]},
     )
-#{local_secondary_indexes}#{global_secondary_indexes}#{stream_specification}
+
+    billing_mode #{table[:billing_mode].inspect}
+#{local_secondary_indexes}#{global_secondary_indexes}#{stream_specification}#{scalable_targets}#{scaling_policies}
   end
         EOS
       end

data/lib/dyna/dsl/table.rb

@@ -11,6 +11,8 @@ module Dyna
 
           @result = Hashie::Mash.new({
             :table_name => table_name,
+            :scalable_targets => [],
+            :scaling_policies => [],
           })
           instance_eval(&block)
         end
@@ -69,6 +71,35 @@ module Dyna
           }.merge(index.result.symbolize_keys)
         end
 
+        def billing_mode(billing_mode)
+          @result.billing_mode = billing_mode
+        end
+
+        def scalable_target(scalable_dimension:, min_capacity:, max_capacity:)
+          @result.scalable_targets << {
+            service_namespace: 'dynamodb',
+            scalable_dimension: scalable_dimension,
+            resource_id: "table/#{@result.table_name}",
+            min_capacity: min_capacity,
+            max_capacity: max_capacity,
+          }
+        end
+
+        def scaling_policy(scalable_dimension:, target_tracking_scaling_policy_configuration:)
+          predefined_metric_type = 'DynamoDBWriteCapacityUtilization'
+          if scalable_dimension == 'dynamodb:table:ReadCapacityUnits'
+            predefined_metric_type = 'DynamoDBReadCapacityUtilization'
+          end
+          @result.scaling_policies << {
+            policy_name: "#{predefined_metric_type}:table/#{@result.table_name}",
+            policy_type: 'TargetTrackingScaling',
+            resource_id: "table/#{@result.table_name}",
+            scalable_dimension: scalable_dimension,
+            service_namespace: 'dynamodb',
+            target_tracking_scaling_policy_configuration: target_tracking_scaling_policy_configuration.merge(predefined_metric_specification: {predefined_metric_type: predefined_metric_type}),
+          }
+        end
+
         class LocalSecondaryIndex
           attr_accessor :result
 

data/lib/dyna/exporter.rb

@@ -15,12 +15,16 @@ module Dyna
     end
 
     def export
-      @ddb.list_tables.table_names
-        .reject { |name| should_skip(name) }
-        .sort
-        .each_with_object({}) do |table_name, result|
-        result[table_name] = self.class.export_table(@ddb, table_name)
+      result = {}
+      @ddb.list_tables.map do |tables|
+        tables.table_names
+          .reject { |name| should_skip(name) }
+          .sort
+          .each do |table_name|
+          result[table_name] = self.class.export_table(@ddb, table_name)
+        end
       end
+      result
     end
 
     def self.table_definition(describe_table)
@@ -28,6 +32,7 @@ module Dyna
         table_name:               describe_table.table_name,
         key_schema:               key_schema(describe_table),
         attribute_definitions:    attribute_definitions(describe_table),
+        billing_mode:             describe_table.billing_mode_summary&.billing_mode,
         provisioned_throughput:   {
           read_capacity_units:    describe_table.provisioned_throughput.read_capacity_units,
           write_capacity_units:   describe_table.provisioned_throughput.write_capacity_units,
@@ -35,9 +40,15 @@ module Dyna
         local_secondary_indexes:  local_secondary_indexes(describe_table),
         global_secondary_indexes: global_secondary_indexes(describe_table),
         stream_specification:     stream_specification(describe_table),
+        scalable_targets:         scalable_targets(describe_table),
+        scaling_policies:         scaling_policies(describe_table),
       }
     end
 
+    def self.aas(aas)
+      @aas = aas
+    end
+
     private
     def self.export_table(ddb, table_name)
       describe_table = ddb.describe_table(table_name: table_name).table
@@ -102,5 +113,43 @@ module Dyna
         stream_view_type: stream_spec.stream_view_type,
       }
     end
+
+    def self.scalable_targets(table)
+      scalable_targets_by_resource_id["table/#{table.table_name}"] || []
+    end
+
+    def self.scalable_targets_by_resource_id
+      return @scalable_targets_by_resource_id if @scalable_targets_by_resource_id
+
+      results = []
+      next_token = nil
+      begin
+        resp = @aas.describe_scalable_targets(service_namespace: 'dynamodb', next_token: next_token)
+        resp.scalable_targets.each do |target|
+          results.push(target)
+        end
+        next_token = resp.next_token
+      end while next_token
+      @scalable_targets_by_resource_id = results.group_by(&:resource_id)
+    end
+
+    def self.scaling_policies(table)
+      scaling_policies_by_resource_id["table/#{table.table_name}"] || []
+    end
+
+    def self.scaling_policies_by_resource_id
+      return @scaling_policies_by_resource_id if @scaling_policies_by_resource_id
+
+      results = []
+      next_token = nil
+      begin
+        resp = @aas.describe_scaling_policies(service_namespace: 'dynamodb', next_token: next_token)
+        resp.scaling_policies.each do |policy|
+          results.push(policy)
+        end
+        next_token = resp.next_token
+      end while next_token
+      @scaling_policies_by_resource_id = results.group_by(&:resource_id)
+    end
   end
 end

data/lib/dyna/utils.rb

@@ -1,6 +1,37 @@
 module Dyna
   class Utils
     class << self
+      def normalize_hash(hash)
+        hash.dup.each do |k, v|
+          if v.kind_of?(Array)
+            if v.first.kind_of?(Hash)
+              hash[k] = v.map { |o| normalize_hash(o) }
+            elsif v.first.respond_to?(:to_h)
+              hash[k] = v.map { |o| normalize_hash(o.to_h) }
+            else
+              v.sort!
+            end
+          elsif v.respond_to?(:to_h)
+            hash[k] = normalize_hash(v.to_h)
+          end
+        end
+        sort_keys(hash)
+      end
+
+      def sort_keys(hash)
+        hash = Hash[hash.sort]
+        hash.each do |k, v|
+          if v.kind_of?(Array)
+            if v.first.kind_of?(Hash)
+              hash[k] = v.map { |h| sort_keys(h) }
+            end
+          elsif v.kind_of?(Hash)
+            hash[k] = sort_keys(v)
+          end
+        end
+        hash
+      end
+
       def diff(obj1, obj2, options = {})
         diffy = Diffy::Diff.new(
           obj1.pretty_inspect,

data/lib/dyna/version.rb

@@ -1,3 +1,3 @@
 module Dyna
-  VERSION = '0.1.9'
+  VERSION = '0.2.0'
 end

data/lib/dyna/wrapper/dynamo_db_wrapper.rb

@@ -8,17 +8,22 @@ module Dyna
     end
 
     def tables
-      @ddb.list_tables.table_names.map do |table_name|
-        describe_table = @ddb.describe_table(table_name: table_name).table
-        Table.new(@ddb, describe_table, @options)
-      end
+      @ddb.list_tables.map { |tables|
+        tables.table_names.map do |table_name|
+          describe_table = @ddb.describe_table(table_name: table_name).table
+          Table.new(@ddb, describe_table, @options)
+        end
+      }.flatten
     end
 
     def create(dsl)
       log(:info, 'Create Table', :cyan, "#{dsl.table_name}")
 
       unless @options.dry_run
-        result = @ddb.create_table(dsl.symbolize_keys)
+        params = dsl.symbolize_keys
+        params.delete(:scalable_targets)
+        params.delete(:scaling_policies)
+        result = @ddb.create_table(params)
         @options.updated = true
         result
       end

data/lib/dyna/wrapper/table.rb

@@ -20,9 +20,8 @@ module Dyna
       end
 
       def update(dsl)
-        unless provisioned_throughput_eql?(dsl)
-          wait_until_table_is_active
-          update_table(dsl_provisioned_throughput(dsl))
+        unless billing_mode_eql?(dsl) && provisioned_throughput_eql?(dsl)
+          update_table(dsl)
         end
         unless global_secondary_indexes_eql?(dsl)
           wait_until_table_is_active
@@ -32,6 +31,9 @@ module Dyna
           wait_until_table_is_active
           update_stream_specification(dsl_stream_specification(dsl))
         end
+        unless auto_scaling_eql?(dsl)
+          update_auto_scaling(dsl)
+        end
       end
 
       def delete
@@ -63,14 +65,54 @@ module Dyna
       end
 
       private
+      def auto_scaling_eql?(dsl)
+        scalable_targets_eql?(dsl) && scaling_policies_eql?(dsl)
+      end
+
+      def scalable_targets_eql?(dsl)
+        df = definition[:scalable_targets].map do |target|
+          cmp = target.to_h
+          cmp.delete(:creation_time)
+          cmp.delete(:role_arn)
+          Dyna::Utils.normalize_hash(cmp)
+        end
+        df.sort_by {|s| s[:scalable_dimension] } == dsl[:scalable_targets].map { |target| Dyna::Utils.normalize_hash(target) }.sort_by {|s| s[:scalable_dimension] }
+      end
+
+      def scaling_policies_for_diff
+        definition[:scaling_policies].map { |policy|
+          #Dyna::Utils.normalize_hash({target_tracking_scaling_policy_configuration: {target_value: policy.target_tracking_scaling_policy_configuration.target_value} })
+          cmp = policy.to_h
+          cmp.delete(:alarms)
+          cmp.delete(:creation_time)
+          cmp.delete(:policy_arn)
+          Dyna::Utils.normalize_hash(cmp)
+        }.sort_by {|s| s[:scalable_dimension] }
+      end
+
+      def scaling_policies_eql?(dsl)
+        scaling_policies_for_diff == dsl.scaling_policies.map { |policy| Dyna::Utils.normalize_hash(policy) }.sort_by {|s| s[:scalable_dimension] }
+      end
+
       def definition_eql?(dsl)
         definition == dsl.definition
       end
 
       def provisioned_throughput_eql?(dsl)
+        if definition[:billing_mode] == 'PROVISIONED' && billing_mode_eql?(dsl)
+          return true
+        end
         self_provisioned_throughput == dsl_provisioned_throughput(dsl)
       end
 
+      def billing_mode_eql?(dsl)
+        if definition[:billing_mode] == dsl[:billing_mode]
+          return true
+        end
+
+        definition[:billing_mode].nil? && dsl[:billing_mode].to_s == 'PROVISIONED'
+      end
+
       def self_provisioned_throughput
         definition.select {|k,v| k == :provisioned_throughput}
       end
@@ -169,11 +211,24 @@ module Dyna
       end
 
       def update_table(dsl)
-        log(:info, "  table: #{@table.table_name}\n".green + Dyna::Utils.diff(self_provisioned_throughput, dsl, :color => @options.color, :indent => '    '), false)
+        params = {}
+        df_params = {}
+        unless billing_mode_eql?(dsl)
+          params[:billing_mode] = dsl[:billing_mode]
+          df_params[:billing_mode] = definition[:billing_mode]
+        end
+
+        if provisioned_throughput_eql?(dsl) == false && dsl[:scalable_targets].empty?
+          params[:provisioned_throughput] = dsl[:provisioned_throughput].symbolize_keys
+          df_params[:provisioned_throughput] = self_provisioned_throughput[:provisioned_throughput]
+        end
+
+        return if params.empty?
+        log(:info, "  table: #{@table.table_name}\n".green + Dyna::Utils.diff(df_params, params, :color => @options.color, :indent => '    '), false)
         unless @options.dry_run
-          params = dsl.dup
+          wait_until_table_is_active
           params[:table_name] = @table.table_name
-          @ddb.update_table(params)
+          @ddb.update_table(params.symbolize_keys)
           @options.updated = true
         end
       end
@@ -205,6 +260,48 @@ module Dyna
           @options.updated = true
         end
       end
+
+      def update_auto_scaling(dsl)
+        has_change = false
+        unless scalable_targets_eql?(dsl)
+          has_change = true
+          df_cmp = definition[:scalable_targets].sort_by { |target| target[:scalable_dimension] }.map do |target|
+            h = target.to_h
+            h.delete(:creation_time)
+            h.delete(:role_arn)
+            Dyna::Utils.normalize_hash(h)
+          end
+          dsl_cmp = dsl.scalable_targets.sort_by { |target| target[:scalable_dimension] }.map { |target| Dyna::Utils.normalize_hash(target) }
+          log(:info, "  table: #{@table.table_name}(update scalable targets)\n".green + Dyna::Utils.diff(df_cmp, dsl_cmp, :color => @options.color, :indent => '    '), false)
+        end
+
+        unless scaling_policies_eql?(dsl)
+          has_change = true
+          dsl_cmp = dsl.scaling_policies.map { |policy| Dyna::Utils.normalize_hash(policy) }.sort_by {|s| s[:scalable_dimension] }
+          log(:info, "  table: #{@table.table_name}(update scaling policies)\n".green + Dyna::Utils.diff(scaling_policies_for_diff, dsl_cmp, :color => @options.color, :indent => '    '), false)
+        end
+
+        unless @options.dry_run
+          if has_change
+            definition[:scalable_targets].each do |target|
+              @options.aas.deregister_scalable_target(
+                service_namespace: 'dynamodb',
+                resource_id: target.resource_id,
+                scalable_dimension: target.scalable_dimension,
+              )
+            end
+
+            dsl.scalable_targets.each do |target|
+              @options.aas.register_scalable_target(target)
+            end
+
+            dsl.scaling_policies.each do |policy|
+              @options.aas.put_scaling_policy(policy)
+            end
+          end
+          @options.updated = true
+        end
+      end
     end
   end
 end

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: dyna
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.0
 platform: ruby
 authors:
 - wata
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-02 00:00:00.000000000 Z
+date: 2019-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-dynamodb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-applicationautoscaling
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
   requirement: !ruby/object:Gem::Requirement
@@ -174,7 +188,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Codenize DynamoDB table