dyna 0.1.1

data/lib/dyna/dsl.rb

@@ -0,0 +1,54 @@
+module Dyna
+  class DSL
+    include Dyna::TemplateHelper
+
+    class << self
+      def define(source, path)
+        self.new(path) do
+          eval(source, binding, path)
+        end
+      end
+
+      def convert(region, exported)
+        Converter.convert(region, exported)
+      end
+    end
+
+    attr_reader :result
+
+    def initialize(path, &block)
+      @path = path
+      @result = OpenStruct.new(:ddbs => {})
+
+      @context = Hashie::Mash.new(
+        :path      => path,
+        :templates => {},
+      )
+
+      instance_eval(&block)
+    end
+
+    private
+    def template(name, &block)
+      @context.templates[name.to_s] = block
+    end
+
+    def require(file)
+      tablefile = (file =~ %r|\A/|) ? file : File.expand_path(File.join(File.dirname(@path), file))
+
+      if File.exist?(tablefile)
+        instance_eval(File.read(tablefile), tablefile)
+      elsif File.exist?(tablefile + '.rb')
+        instance_eval(File.read(tablefile + '.rb'), tablefile + '.rb')
+      else
+        Kernel.require(file)
+      end
+    end
+
+    def dynamo_db(region, &block)
+      ddb = @result.ddbs[region]
+      tables = ddb ? ddb.tables : []
+      @result.ddbs[region] = DynamoDB.new(@context, tables, &block).result
+    end
+  end
+end

data/lib/dyna/exporter.rb

@@ -0,0 +1,105 @@
+module Dyna
+  class Exporter
+    include Filterable
+
+    class << self
+      def export(ddb, options = {})
+        self.new(ddb, options).export
+      end
+    end
+
+    def initialize(ddb, options = {})
+      @ddb = ddb
+      @options = options
+    end
+
+    def export
+      @ddb.list_tables.table_names
+        .reject { |name| should_skip(name) }
+        .sort
+        .each_with_object({}) do |table_name, result|
+        result[table_name] = self.class.export_table(@ddb, table_name)
+      end
+    end
+
+    def self.table_definition(describe_table)
+      {
+        table_name:               describe_table.table_name,
+        key_schema:               key_schema(describe_table),
+        attribute_definitions:    attribute_definitions(describe_table),
+        provisioned_throughput:   {
+          read_capacity_units:    describe_table.provisioned_throughput.read_capacity_units,
+          write_capacity_units:   describe_table.provisioned_throughput.write_capacity_units,
+        },
+        local_secondary_indexes:  local_secondary_indexes(describe_table),
+        global_secondary_indexes: global_secondary_indexes(describe_table),
+        stream_specification:     stream_specification(describe_table),
+      }
+    end
+
+    private
+    def self.export_table(ddb, table_name)
+      describe_table = ddb.describe_table(table_name: table_name).table
+      table_definition(describe_table)
+    end
+
+    def self.key_schema(table)
+      table.key_schema.map do |schema|
+        {
+          attribute_name: schema.attribute_name,
+          key_type:       schema.key_type,
+        }
+      end
+    end
+
+    def self.attribute_definitions(table)
+      table.attribute_definitions.map do |definition|
+        {
+          attribute_name: definition.attribute_name,
+          attribute_type: definition.attribute_type,
+        }
+      end
+    end
+
+    def self.global_secondary_indexes(table)
+      return nil unless table.global_secondary_indexes
+      table.global_secondary_indexes.map do |index|
+        {
+          index_name:             index.index_name,
+          key_schema:             key_schema(index),
+          projection:             {
+            projection_type:      index.projection.projection_type,
+            non_key_attributes:   index.projection.non_key_attributes,
+          },
+          provisioned_throughput: {
+            read_capacity_units:  index.provisioned_throughput.read_capacity_units,
+            write_capacity_units: index.provisioned_throughput.write_capacity_units,
+          },
+        }
+      end
+    end
+
+    def self.local_secondary_indexes(table)
+      return nil unless table.local_secondary_indexes
+      table.local_secondary_indexes.map do |index|
+        {
+          index_name:           index.index_name,
+          key_schema:           key_schema(index),
+          projection:           {
+            projection_type:    index.projection.projection_type,
+            non_key_attributes: index.projection.non_key_attributes,
+          },
+        }
+      end
+    end
+
+    def self.stream_specification(table)
+      stream_spec = table.stream_specification
+      return nil unless stream_spec
+      {
+        stream_enabled: stream_spec.stream_enabled,
+        stream_view_type: stream_spec.stream_view_type,
+      }
+    end
+  end
+end

data/lib/dyna/ext/hash-ext.rb

@@ -0,0 +1,12 @@
+class Hash
+  def symbolize_keys
+    self.each_with_object({}) do |(k, v), h|
+      h[k.to_s.to_sym] = (v.is_a?(Hash) ? v.symbolize_keys : v)
+      if v.is_a?(Array)
+        h[k.to_s.to_sym] = v.each_with_object([]) do |h2, a|
+          a << (h2.is_a?(Hash) ? h2.symbolize_keys : h2)
+        end
+      end
+    end
+  end
+end

data/lib/dyna/ext/string-ext.rb

@@ -0,0 +1,25 @@
+class String
+  @@colorize = false
+
+  class << self
+    def colorize=(value)
+      @@colorize = value
+    end
+
+    def colorize
+      @@colorize
+    end
+  end # of class methods
+
+  Term::ANSIColor::Attribute.named_attributes.map do |attribute|
+    class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+      def #{attribute.name}
+        if @@colorize
+          Term::ANSIColor.send(#{attribute.name.inspect}, self)
+        else
+          self
+        end
+      end
+    EOS
+  end
+end

data/lib/dyna/filterable.rb

@@ -0,0 +1,21 @@
+module Dyna
+  module Filterable
+    def should_skip(table_name)
+      if @options.table_names
+        unless @options.table_names.include?(table_name)
+          log(:debug, "skip table(with tables_names option) #{table_name}")
+          return true
+        end
+      end
+
+      if @options.exclude_table_names
+        if @options.exclude_table_names.any? {|regex| table_name =~ regex}
+          log(:debug, "skip table(with exclude_tables_names option) #{table_name}")
+          return true
+        end
+      end
+
+      false
+    end
+  end
+end

data/lib/dyna/logger.rb

@@ -0,0 +1,30 @@
+module Dyna
+  class Logger < ::Logger
+    include Singleton
+
+    def initialize
+      super($stdout)
+
+      self.formatter = proc do |severity, datetime, progname, msg|
+        "#{msg}\n"
+      end
+
+      self.level = Logger::INFO
+    end
+
+    def set_debug(value)
+      self.level = value ? Logger::DEBUG : Logger::INFO
+    end
+
+    module ClientHelper
+      def log(level, message, color, log_id = nil)
+        message = "[#{level.to_s.upcase}] #{message}" unless level == :info
+        message << ": #{log_id}" if log_id
+        message << ' (dry-run)' if @options && @options.dry_run
+        logger = (@options && @options.logger) || Dyna::Logger.instance
+        message = message.send(color) if color
+        logger.send(level, message)
+      end
+    end
+  end
+end

data/lib/dyna/template_helper.rb

@@ -0,0 +1,20 @@
+module Dyna
+  module TemplateHelper
+    def include_template(template_name, context = {})
+      tmplt = @context.templates[template_name.to_s]
+
+      unless tmplt
+        raise "Template `#{template_name}` is not defined"
+      end
+
+      context_orig = @context
+      @context = @context.merge(context)
+      instance_eval(&tmplt)
+      @context = context_orig
+    end
+
+    def context
+      @context
+    end
+  end
+end

data/lib/dyna/utils.rb

@@ -0,0 +1,17 @@
+module Dyna
+  class Utils
+    class << self
+      def diff(obj1, obj2, options = {})
+        diffy = Diffy::Diff.new(
+          obj1.pretty_inspect,
+          obj2.pretty_inspect,
+          :diff => '-u'
+        )
+
+        out = diffy.to_s(options[:color] ? :color : :text).gsub(/\s+\z/m, '')
+        out.gsub!(/^/, options[:indent]) if options[:indent]
+        out
+      end
+    end
+  end
+end

data/lib/dyna/version.rb

@@ -0,0 +1,3 @@
+module Dyna
+  VERSION = '0.1.1'
+end

data/lib/dyna/wrapper/dynamo_db_wrapper.rb

@@ -0,0 +1,31 @@
+module Dyna
+  class DynamoDBWrapper
+    include Logger::ClientHelper
+
+    def initialize(ddb, options)
+      @ddb = ddb
+      @options = options.dup
+    end
+
+    def tables
+      @ddb.list_tables.table_names.map do |table_name|
+        describe_table = @ddb.describe_table(table_name: table_name).table
+        Table.new(@ddb, describe_table, @options)
+      end
+    end
+
+    def create(dsl)
+      log(:info, 'Create Table', :cyan, "#{dsl.table_name}")
+
+      unless @options.dry_run
+        result = @ddb.create_table(dsl.symbolize_keys)
+        @options.updated = true
+        result
+      end
+    end
+
+    def updated?
+      !!@options.updated
+    end
+  end
+end

data/lib/dyna/wrapper/table.rb

@@ -0,0 +1,205 @@
+module Dyna
+  class DynamoDBWrapper
+    class Table
+      extend Forwardable
+      include Logger::ClientHelper
+
+      def_delegators(
+        :@table,
+        :table_name
+      )
+
+      def initialize(ddb, table, options)
+        @ddb = ddb
+        @table = table
+        @options = options
+      end
+
+      def eql?(dsl)
+        definition_eql?(dsl)
+      end
+
+      def update(dsl)
+        unless provisioned_throughput_eql?(dsl)
+          wait_until_table_is_active
+          update_table(dsl_provisioned_throughput(dsl))
+        end
+        unless global_secondary_indexes_eql?(dsl)
+          wait_until_table_is_active
+          update_table_index(dsl, dsl_global_secondary_index_updates(dsl))
+        end
+        unless stream_specification_eql?(dsl)
+          wait_until_table_is_active
+          update_stream_specification(dsl_stream_specification(dsl))
+        end
+      end
+
+      def delete
+        log(:info, 'Delete Table', :red, "#{table_name}")
+        
+        unless @options.dry_run
+          @ddb.delete_table(table_name: @table.table_name)
+          @options.updated = true
+        end
+      end
+
+      def definition
+        Exporter.table_definition(@table).symbolize_keys
+      end
+
+      def wait_until_table_is_active
+        log(:info, "waiting table #{@table.table_name} to be ACTIVE or deleted..", false)
+        loop do
+          begin
+            desc = @ddb.describe_table(table_name: table_name).table
+          rescue => e
+            break
+          end
+          status = desc.table_status
+          log(:info, "status... #{status}", false)
+          break if desc.table_status == 'ACTIVE'
+          sleep 3
+        end
+      end
+
+      private
+      def definition_eql?(dsl)
+        definition == dsl.definition
+      end
+
+      def provisioned_throughput_eql?(dsl)
+        self_provisioned_throughput == dsl_provisioned_throughput(dsl)
+      end
+
+      def self_provisioned_throughput
+        definition.select {|k,v| k == :provisioned_throughput}
+      end
+
+      def dsl_provisioned_throughput(dsl)
+        dsl.symbolize_keys.select {|k,v| k == :provisioned_throughput}
+      end
+
+      def global_secondary_indexes_eql?(dsl)
+        self_global_secondary_indexes == dsl_global_secondary_indexes(dsl)
+      end
+
+      def self_global_secondary_indexes
+        definition[:global_secondary_indexes]
+      end
+
+      def dsl_global_secondary_indexes(dsl)
+        dsl.symbolize_keys[:global_secondary_indexes]
+      end
+
+      def dsl_global_secondary_index_updates(dsl)
+        actual_by_name = (self_global_secondary_indexes || {}).group_by { |index| index[:index_name] }.each_with_object({}) do |(k, v), h|
+          h[k] = v.first
+        end
+        expect_by_name = (dsl_global_secondary_indexes(dsl) || {}).group_by { |index| index[:index_name] }.each_with_object({}) do |(k, v), h|
+          h[k] = v.first
+        end
+        params = []
+        expect_by_name.each do |index_name, expect_index|
+          actual_index = actual_by_name[index_name]
+          unless actual_index
+            unless params.empty?
+              log(:warn, 'Can not add multiple GSI at once', :yellow, index_name)
+              next
+            end
+            params << {create: expect_index}
+          end
+        end
+
+        expect_by_name.each do |index_name, expect_index|
+          actual_index = actual_by_name.delete(index_name)
+          if actual_index != nil &&
+            actual_index[:provisioned_throughput] != expect_index[:provisioned_throughput]
+            if params.any? { |param| param[:update] }
+              log(:warn, 'Can not update multiple GSI at once', :yellow, index_name)
+              next
+            end
+            params << {update: {
+              index_name: index_name,
+              provisioned_throughput: expect_index[:provisioned_throughput]
+            }}
+          end
+        end
+
+        actual_by_name.each do |index_name, actual_index|
+          if params.any? { |param| param[:delete] }
+              log(:warn, 'Can not delete multiple GSI at once', :yellow, index_name)
+              next
+          end
+          params << {delete: { index_name: index_name }}
+        end
+
+        params
+      end
+
+      def stream_specification_eql?(dsl)
+        actual = self_stream_specification
+        expect = dsl_stream_specification(dsl)
+        if (actual == nil || actual[:stream_specification] == nil) &&
+           (expect == nil || expect[:stream_specification] == nil || expect[:stream_specification][:stream_enabled] == false)
+          return true
+        end
+        actual == expect
+      end
+
+      def self_stream_specification
+        definition.select {|k,v| k == :stream_specification}
+      end
+
+      def dsl_stream_specification(dsl)
+        dsl.symbolize_keys.select {|k,v| k == :stream_specification}
+      end
+
+      def update_stream_specification(dsl)
+        log(:info, "  table: #{@table.table_name}(update stream spec)\n".green + Dyna::Utils.diff(self_stream_specification, dsl, :color => @options.color, :indent => '    '), false)
+        unless @options.dry_run
+          params = { table_name: @table.table_name }.merge(dsl)
+          @ddb.update_table(params)
+          @options.updated = true
+        end
+      end
+
+      def update_table(dsl)
+        log(:info, "  table: #{@table.table_name}\n".green + Dyna::Utils.diff(self_provisioned_throughput, dsl, :color => @options.color, :indent => '    '), false)
+        unless @options.dry_run
+          params = dsl.dup
+          params[:table_name] = @table.table_name
+          @ddb.update_table(params)
+          @options.updated = true
+        end
+      end
+
+      def update_table_index(dsl, index_params)
+        log(:info, "  table: #{@table.table_name}(update GSI)".green, false)
+        index_params.each do |index_param|
+          if index_param[:create]
+            log(:info, "  index: #{index_param[:create][:index_name]}(create GSI)".cyan, false)
+            log(:info, "    => #{index_param[:create]}".cyan, false)
+          end
+          if index_param[:update]
+            log(:info, "  index: #{index_param[:update][:index_name]}(update GSI)".green, false)
+            log(:info, "    => #{index_param[:update]}".green, false)
+          end
+          if index_param[:delete]
+            log(:info, "  index: #{index_param[:delete][:index_name]}(delete GSI)".red, false)
+            log(:info, "    => #{index_param[:delete]}".red, false)
+          end
+        end
+
+        unless @options.dry_run
+          params = {
+            table_name: @table.table_name,
+            attribute_definitions: dsl.symbolize_keys[:attribute_definitions],
+            global_secondary_index_updates: index_params,
+          }
+          @ddb.update_table(params)
+          @options.updated = true
+        end
+      end
+    end
+  end
+end

data/lib/dyna.rb

@@ -0,0 +1,27 @@
+require 'forwardable'
+require 'logger'
+require 'term/ansicolor'
+require 'diffy'
+require 'hashie'
+require 'singleton'
+
+require 'aws-sdk'
+
+require 'dyna/version'
+require 'dyna/logger'
+require 'dyna/filterable'
+require 'dyna/client'
+require 'dyna/exporter'
+require 'dyna/template_helper'
+require 'dyna/dsl'
+require 'dyna/utils'
+require 'dyna/dsl/converter'
+require 'dyna/dsl/dynamo_db'
+require 'dyna/dsl/table'
+require 'dyna/ext/string-ext'
+require 'dyna/ext/hash-ext'
+require 'dyna/wrapper/table'
+require 'dyna/wrapper/dynamo_db_wrapper'
+
+module Dyna
+end