dyna 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 468dd97763a9c340c77d46800af940b47bc6ac22
+  data.tar.gz: 6f446670b9ece4b36d7056dd0c2e3d61f380ebdd
+SHA512:
+  metadata.gz: 011fd23c5040bb08d12e090669231e1db2de7ec899993456ad603e26c44080f345bad620ef86ad38a63569d9575b0301eacd5ac955a759760cb2787a5fd850ed
+  data.tar.gz: e336f511705841d336aabd8c3599a447bc3323aba06f300caa78f41d853eeaa93e83849d0d80968cc43f61ef30ba4f3e6a8e7dd39b2e4596047e4352af48e6f4

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in dyna.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 shinya-watanabe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# Dyna
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/dyna`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'dyna'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dyna
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/dyna.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec
+

data/bin/dyna

@@ -0,0 +1,191 @@
+#!/usr/bin/env ruby
+$: << File.expand_path("#{File.dirname __FILE__}/../lib")
+require 'rubygems'
+require 'dyna'
+require 'optparse'
+require 'pry-byebug'
+
+Version = Dyna::VERSION
+
+mode = nil
+file = 'Dynafile'
+output_file = '-'
+split = false
+MAGIC_COMMENT = <<-EOS
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+EOS
+
+options = {
+  :dry_run => false,
+  :format  => :ruby,
+  :color   => true,
+  :debug   => false,
+}
+
+ARGV.options do |opt|
+  begin
+    access_key = nil
+    secret_key = nil
+    region = nil
+    profile_name = nil
+    credentials_path = nil
+    format_passed = false
+    role_arn = nil
+    serial_number = nil
+    token_code = nil
+    role_session_name = nil
+
+    opt.on('-p', '--profile PROFILE_NAME')                   {|v| profile_name                  = v      }
+    opt.on('',   '--role_arn ROLE_ARN')                      {|v| role_arn                      = v      }
+    opt.on('',   '--serial_number SERIAL_NUMBER')            {|v| serial_number                 = v      }
+    opt.on('',   '--token_code TOKEN_CODE')                  {|v| token_code                    = v      }
+    opt.on('',   '--role_session_name ROLE_SESSION_NAME')    {|v| role_session_name             = v      }
+    opt.on(''  , '--credentials-path PATH')                  {|v| credentials_path              = v      }
+    opt.on('-k', '--access-key ACCESS_KEY')                  {|v| access_key                    = v      }
+    opt.on('-s', '--secret-key SECRET_KEY')                  {|v| secret_key                    = v      }
+    opt.on('-r', '--region REGION')                          {|v| region                        = v      }
+    opt.on('-a', '--apply')                                  {|v| mode                          = :apply }
+    opt.on('-f', '--file FILE')                              {|v| file                          = v      }
+    opt.on('',   '--table_names TABLE_NAMES', Array)         {|v| options[:table_names]         = v      }
+    opt.on('',   '--exclude_table_names TABLE_NAMES', Array) {|v| options[:exclude_table_names] = v      }
+    opt.on('',   '--dry-run')                                {|v| options[:dry_run]             = true   }
+    opt.on('-e', '--export')                                 {|v| mode                          = :export}
+    opt.on('-o', '--output FILE')                            {|v| output_file                   = v      }
+    opt.on('',   '--split')                                  {|v| split                         = true   }
+    opt.on('',   '--split-more')                             {|v| split                         = :more  }
+    opt.on(''  , '--no-color')                               {    options[:color]               = false  }
+    opt.on(''  , '--debug')                                  {    options[:debug]               = true   }
+    opt.parse!
+
+    aws_opts = {}
+    if access_key and secret_key
+      aws_opts = {
+        :access_key_id     => access_key,
+        :secret_access_key => secret_key,
+      }
+    elsif profile_name or credentials_path
+      credentials_opts = {}
+      credentials_opts[:profile_name] = profile_name if profile_name
+      credentials_opts[:path] = credentials_path if credentials_path
+      provider = AWS::Core::CredentialProviders::SharedCredentialFileProvider.new(credentials_opts)
+      aws_opts[:credential_provider] = provider
+    elsif role_arn and serial_number
+      credentials = Aws::AssumeRoleCredentials.new(
+        client: Aws::STS::Client.new,
+        role_arn: role_arn,
+        role_session_name: role_session_name,
+        serial_number: serial_number,
+        token_code: token_code,
+      )
+      aws_opts[:credentials] = credentials
+    elsif (access_key and !secret_key) or (!access_key and secret_key) or mode.nil?
+      puts opt.help
+      exit 1
+    end
+
+    aws_opts[:region] = region if region
+    Aws.config.update(aws_opts)
+
+    # Remap groups to exclude to regular expressions (if they're surrounded by '/')
+    if options[:exclude_table_names]
+      options[:exclude_table_names].map! do |name|
+        name =~ /\A\/(.*)\/\z/ ? Regexp.new($1) : Regexp.new("\A#{Regexp.escape(name)}\z")
+      end
+    end
+  rescue => e
+    $stderr.puts("[ERROR] #{e.message}")
+    exit 1
+  end
+end
+
+String.colorize = options[:color]
+
+if options[:debug]
+  Aws.config.update({
+    :http_wire_trace => true,
+    :logger => Dyna::Logger.instance,
+  })
+end
+
+begin
+  logger = Dyna::Logger.instance
+  logger.set_debug(options[:debug])
+  client = Dyna::Client.new(options)
+
+  case mode
+  when :export
+    if split
+      logger.info('Export Table')
+
+      output_file = 'Dynafile' if output_file == '-'
+      requires = []
+      base_dir = File.dirname(output_file)
+
+      client.export(options) do |exported, converter|
+        write_table_file = proc do |table_file, tables|
+          requires << table_file
+
+          logger.info("  write `#{table_file}`")
+          FileUtils.mkdir_p(File.dirname(table_file))
+
+          open(table_file, 'wb') do |f|
+            f.puts MAGIC_COMMENT
+            f.puts converter.call(tables)
+          end
+        end
+
+        exported.each do |table_name, tables|
+          table_file = File.join(base_dir, "#{Aws::DynamoDB::Client.new.config.region}/#{table_name}.dyna")
+          write_table_file.call(table_file, table_name => tables)
+        end
+      end
+
+      logger.info("  write `#{output_file}`")
+      open(output_file, 'wb') do |f|
+        f.puts MAGIC_COMMENT
+
+        requires.each do |table_file|
+          table_file.sub!(%r|\A#{Regexp.escape base_dir}/?|, '')
+          f.puts "require '#{table_file}'"
+        end
+      end
+    else
+      exported = client.export(options)
+
+      if output_file == '-'
+        logger.info('# Export Table') if options[:format] == :ruby
+        puts exported
+      else
+        logger.info("Export Table to `#{output_file}`")
+
+        open(output_file, 'wb') do |f|
+          f.puts MAGIC_COMMENT if options[:format] == :ruby
+          f.puts exported
+        end
+      end
+    end
+  when :apply
+    unless File.exist?(file)
+      raise "No Dynafile found (looking for: #{file})"
+    end
+
+    msg = "Apply `#{file}` to DynamoDB"
+    msg << ' (dry-run)' if options[:dry_run]
+    logger.info(msg)
+
+    updated = client.apply(file)
+
+    logger.info('No change'.intense_blue) unless updated
+  else
+    raise 'must not happen'
+  end
+rescue => e
+  if options[:debug]
+    raise e
+  else
+    $stderr.puts("[ERROR] #{e.message}".red)
+    $stderr.puts("#{e.backtrace.join("\n")}".red)
+    exit 1
+  end
+end

data/dyna.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dyna/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dyna"
+  spec.version       = Dyna::VERSION
+  spec.authors       = ["wata"]
+  spec.email         = ["wata.gm@gmail.com"]
+
+  spec.summary       = %q{Codenize DynamoDB table}
+  spec.description   = %q{Manager DynamoDB table by DSL}
+  spec.homepage      = 'https://github.com/wata-gh/dyna'
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_dependency 'aws-sdk', '~> 2'
+  spec.add_dependency 'term-ansicolor', '~> 1.4'
+  spec.add_dependency 'diffy', '~> 3.1'
+  spec.add_dependency 'hashie', '~> 3.4'
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry-byebug', '~> 3.4'
+end

data/lib/dyna/client.rb

@@ -0,0 +1,89 @@
+module Dyna
+  class Client
+    include Logger::ClientHelper
+    include Filterable
+
+    def initialize(options = {})
+      @options = OpenStruct.new(options)
+      @options_hash = options
+      @options.ddb = Aws::DynamoDB::Client.new
+    end
+
+    def apply(file)
+      walk(file)
+    end
+
+    def export(options = {})
+      exported = Exporter.export(@options.ddb, @options)
+
+      converter = proc do |src|
+        DSL.convert(@options.ddb.config.region, src)
+      end
+
+      if block_given?
+        yield(exported, converter)
+      else
+        converter.call(exported)
+      end
+    end
+
+    private
+    def load_file(file)
+      if file.kind_of?(String)
+        open(file) do |f|
+          parse(f.read, file)
+        end
+      elsif file.respond_to?(:read)
+        parse(file.read, file.path)
+      else
+        raise TypeError, "can't load #{file}"
+      end
+    end
+
+    def parse(src, path)
+      DSL.define(src, path).result
+    end
+
+    def walk(file)
+      dsl = load_file(file)
+      dsl_ddbs = dsl.ddbs
+      ddb_wrapper = DynamoDBWrapper.new(@options.ddb, @options)
+
+      dsl_ddbs.each do |region, ddb_dsl|
+        walk_ddb(ddb_dsl, ddb_wrapper) if @options.ddb.config.region == region
+      end
+
+      ddb_wrapper.updated?
+    end
+
+    def walk_ddb(ddb_dsl, ddb_wrapper)
+      table_list_dsl = ddb_dsl.tables.group_by(&:table_name).each_with_object({}) do |(k, v), h|
+        h[k] = v.first unless should_skip(k)
+      end
+      table_list_aws = ddb_wrapper.tables.group_by(&:table_name).each_with_object({}) do |(k, v), h|
+        h[k] = v.first unless should_skip(k)
+      end
+
+      table_list_dsl.each do |name, table_dsl|
+        unless table_list_aws[name]
+          result = ddb_wrapper.create(table_dsl)
+          if result
+            table_list_aws[name] = DynamoDBWrapper::Table.new(
+              @options.ddb,
+              result.table_description,
+              @options,
+            )
+          end
+        end
+      end
+
+      table_list_dsl.each do |name, table_dsl|
+        table_aws = table_list_aws.delete(name)
+        next unless table_aws # only dry-run and should be created
+        table_aws.update(table_dsl) unless table_aws.eql?(table_dsl)
+      end
+
+      table_list_aws.values.each(&:delete)
+    end
+  end
+end

data/lib/dyna/dsl/converter.rb

@@ -0,0 +1,87 @@
+module Dyna
+  class DSL
+    class Converter
+      class << self
+        def convert(region, exported)
+          self.new(region, exported).convert
+        end
+      end
+
+      def initialize(region, exported)
+        @region = region
+        @exported = exported
+      end
+
+      def convert
+        output_dynamo_db
+      end
+
+      private
+      def output_dynamo_db
+        tables = @exported.map {|name, table|
+          output_table(name, table)
+        }.join("\n").strip
+
+        <<-EOS
+dynamo_db "#{@region}" do
+  #{tables}
+end
+        EOS
+      end
+
+      def output_table(name, table)
+        local_secondary_indexes = ''
+        global_secondary_indexes = ''
+        if table[:local_secondary_indexes]
+          local_secondary_indexes_tmpl = <<-EOS.chomp
+<% table[:local_secondary_indexes].each do |index| %>
+    local_secondary_index <%= index[:index_name].inspect %> do
+      key_schema hash: <%= index[:key_schema][0][:attribute_name].inspect %>, range: <%= index[:key_schema][1][:attribute_name].inspect %><% if index[:projection] %>
+      projection projection_type: <%= index[:projection][:projection_type].inspect %><% end %>
+    end
+<% end %>
+EOS
+          local_secondary_indexes = ERB.new(local_secondary_indexes_tmpl).result(binding)
+        end
+
+        if table[:global_secondary_indexes]
+          p table[:global_secondary_indexes]
+          global_secondary_indexes_tmpl = <<-EOS.chomp
+<% table[:global_secondary_indexes].each do |index| %>
+    global_secondary_index <%= index[:index_name].inspect %> do
+      key_schema hash: <%= index[:key_schema][0][:attribute_name].inspect %><% if index[:key_schema].size == 2 %>, range: <%= index[:key_schema][1][:attribute_name].inspect %><% end %><% if index[:projection] %>
+      projection projection_type: <%= index[:projection][:projection_type].inspect %><% end %>
+      provisioned_throughput read_capacity_units: <%= index[:provisioned_throughput][:read_capacity_units] %>, write_capacity_units: <%= index[:provisioned_throughput][:read_capacity_units] %>
+    end
+<% end %>
+EOS
+          global_secondary_indexes = ERB.new(global_secondary_indexes_tmpl).result(binding)
+        end
+
+        attribute_definitions_tmpl = <<-EOS.chomp
+<% table[:attribute_definitions].each do |attr| %>
+    attribute_definition(
+      attribute_name: <%= attr[:attribute_name].inspect %>,
+      attribute_type: <%= attr[:attribute_type].inspect %>,
+    )
+<% end %>
+EOS
+        attribute_definitions = ERB.new(attribute_definitions_tmpl).result(binding)
+        <<-EOS
+  table "#{name}" do
+    key_schema(
+      hash: #{table[:key_schema][0][:attribute_name].inspect},
+      range: #{table[:key_schema][1][:attribute_name].inspect},
+    )
+#{attribute_definitions}
+    provisioned_throughput(
+      read_capacity_units: #{table[:provisioned_throughput][:read_capacity_units]},
+      write_capacity_units: #{table[:provisioned_throughput][:write_capacity_units]},
+    )
+#{local_secondary_indexes}#{global_secondary_indexes}
+  end
+        EOS
+      end
+    end
+  end
+end

data/lib/dyna/dsl/dynamo_db.rb

@@ -0,0 +1,31 @@
+module Dyna
+  class DSL
+    class DynamoDB
+      include Dyna::TemplateHelper
+
+      attr_reader :result
+
+      def initialize(context, tables, &block)
+        @context = context
+        @result = OpenStruct.new({
+          :tables => tables,
+        })
+
+        instance_eval(&block)
+      end
+
+      private
+      def table(name, &block)
+        if table_names.include?(name)
+          raise "Table `#{name}` is already defined"
+        end
+
+        @result.tables << Table.new(@context, name, &block).result
+      end
+
+      def table_names
+        @result.tables.map(&:table_name)
+      end
+    end
+  end
+end

data/lib/dyna/dsl/table.rb

@@ -0,0 +1,112 @@
+module Dyna
+  class DSL
+    class DynamoDB
+      class Table
+        include Dyna::TemplateHelper
+        attr_reader :result
+
+        def initialize(context, table_name, &block)
+          @table_name = table_name
+          @context = context
+
+          @result = Hashie::Mash.new({
+            :table_name => table_name,
+          })
+          instance_eval(&block)
+        end
+
+        def key_schema(hash:, range: nil)
+          @result.key_schema = [{
+            attribute_name: hash,
+            key_type: 'HASH',
+          }]
+
+          if range
+            @result.key_schema << {
+              attribute_name: range,
+              key_type: 'RANGE',
+            }
+          end
+        end
+
+        def attribute_definition(attribute_name:, attribute_type:)
+          @result.attribute_definitions ||= []
+          @result.attribute_definitions << {
+            attribute_name: attribute_name,
+            attribute_type: attribute_type,
+          }
+        end
+
+        def provisioned_throughput(read_capacity_units:, write_capacity_units:)
+          @result.provisioned_throughput = {
+            read_capacity_units: read_capacity_units,
+            write_capacity_units: write_capacity_units,
+          }
+        end
+
+        def stream_specification(stream_enabled:, stream_view_type: nil)
+          @result.stream_specification = {
+            stream_enabled: stream_enabled,
+            stream_view_type: stream_view_type,
+          }
+        end
+
+        def local_secondary_index(index_name, &block)
+          @result.local_secondary_indexes ||= []
+          index = LocalSecondaryIndex.new
+          index.instance_eval(&block)
+          @result.local_secondary_indexes << {
+            index_name: index_name,
+          }.merge(index.result.symbolize_keys)
+        end
+
+        def global_secondary_index(index_name, &block)
+          @result.global_secondary_indexes ||= []
+          index = GlobalSecondaryIndex.new
+          index.instance_eval(&block)
+          @result.global_secondary_indexes << {
+            index_name: index_name,
+          }.merge(index.result.symbolize_keys)
+        end
+
+        class LocalSecondaryIndex
+          attr_accessor :result
+
+          def initialize
+            @result = Hashie::Mash.new
+          end
+
+          def key_schema(hash:, range: nil)
+            @result.key_schema = [{
+              attribute_name: hash,
+              key_type: 'HASH',
+            }]
+
+            if range
+              @result.key_schema << {
+                attribute_name: range,
+                key_type: 'RANGE',
+              }
+            end
+          end
+
+          def projection(projection_type:, non_key_attributes: nil)
+            @result.projection = {
+              projection_type: projection_type,
+              non_key_attributes: non_key_attributes,
+            }
+          end
+        end
+
+        class GlobalSecondaryIndex < LocalSecondaryIndex
+          def provisioned_throughput(read_capacity_units:, write_capacity_units:)
+            @result.provisioned_throughput = {
+              read_capacity_units: read_capacity_units,
+              write_capacity_units: write_capacity_units,
+            }
+          end
+        end
+      end
+    end
+  end
+end