dynamodb_framework 0.1.0

data/README.md

@@ -0,0 +1,41 @@
+# DynamodbFramework
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/dynamodb_framework`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'dynamodb_framework'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dynamodb_framework
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/dynamodb_framework. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "dynamodb_framework"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/dynamodb_framework.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dynamodb_framework/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dynamodb_framework"
+  spec.version       = DynamodbFramework::VERSION
+  spec.authors       = ["vaughanbrittonsage"]
+  spec.email         = ["vaughanbritton@gmail.com"]
+
+  spec.summary       = 'A lightweight framework to provide managers for working with aws dynamodb (incuding local version).'
+  spec.description   = 'A lightweight framework to provide managers for working with aws dynamodb (incuding local version).'
+  spec.homepage      = "https://github.com/vaughanbrittonsage/dynamodb_framework"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_dependency('json')
+  spec.add_dependency('aws-sdk-core')
+
+end

data/lib/dynamodb_framework/dynamodb_attributes_builder.rb

@@ -0,0 +1,20 @@
+class DynamoDbAttributesBuilder
+  attr_reader :attributes
+
+  def initialize
+    @attributes = []
+  end
+
+  def add(name, type)
+    type_symbol = :S
+    if type == 'number' || type == :N
+      type_symbol = :N
+    elsif type == 'binary' || type == :B
+      type_symbol = :B
+    elsif type == 'bool' || type == 'boolean' || type == :BOOL
+      type_symbol = :BOOL
+    end
+    @attributes.push({ :name => name, :type => type_symbol })
+  end
+
+end

data/lib/dynamodb_framework/dynamodb_migration_manager.rb

@@ -0,0 +1,75 @@
+class DynamoDbMigrationManager
+
+  attr_reader :dynamodb_table_manager
+  attr_reader :dynamodb_repository
+
+  def initialize
+    @dynamodb_repository = DynamoDbRepository.new
+    @dynamodb_table_manager = DynamoDbTableManager.new
+  end
+
+  def connect
+
+    migration_table_name = 'dynamodb_migrations'
+
+    #check if migration table exists
+    if !@dynamodb_table_manager.exists?(migration_table_name)
+      #migration table not found so create it
+      builder = DynamoDbAttributesBuilder.new
+      builder.add(:timestamp, :S)
+      @dynamodb_table_manager.create(migration_table_name, builder.attributes, :timestamp)
+    end
+
+    #set the table name for the repository
+    @dynamodb_repository.table_name = migration_table_name
+
+  end
+
+  def get_last_executed_script
+    scripts = @dynamodb_repository.all()
+    if scripts.length > 0
+      return scripts.sort { |a,b| b['timestamp'] <=> a['timestamp'] }[0]['timestamp']
+    end
+
+    return nil
+  end
+
+  def apply
+
+    last_executed_script = get_last_executed_script()
+
+    scripts = []
+    MigrationScript.descendants.each do |ms|
+      script = ms.new
+      scripts.push(script)
+    end
+
+    scripts.sort { |a,b| a.timestamp <=> b.timestamp }.each do |script|
+      if last_executed_script == nil || script.timestamp > last_executed_script
+        script.apply
+        @dynamodb_repository.put({ :timestamp => script.timestamp })
+      end
+    end
+
+  end
+
+  def rollback
+
+    last_executed_script = get_last_executed_script()
+
+    scripts = []
+    MigrationScript.descendants.each do |ms|
+      script = ms.new
+      scripts.push(script)
+    end
+
+    scripts.sort { |a,b| a.timestamp <=> b.timestamp }.each do |script|
+      if script.timestamp == last_executed_script
+        script.undo
+        @dynamodb_repository.delete({ :timestamp => script.timestamp })
+        return
+      end
+    end
+  end
+
+end

data/lib/dynamodb_framework/dynamodb_migration_script.rb

@@ -0,0 +1,16 @@
+class MigrationScript
+  attr_accessor :timestamp
+
+  def apply
+    raise 'Not implemented.'
+  end
+
+  def undo
+    raise 'Not implemented.'
+  end
+
+  def self.descendants
+    ObjectSpace.each_object(Class).select { |klass| klass < self }
+  end
+
+end

data/lib/dynamodb_framework/dynamodb_repository.rb

@@ -0,0 +1,185 @@
+class DynamoDbRepository
+
+  attr_reader :dynamodb
+  attr_accessor :table_name
+
+  def initialize
+    @dynamodb = DynamoDbStore.new
+  end
+
+  def put(item)
+
+    json_string = item.to_json
+    json_obj = JSON.load(json_string)
+
+    params =
+        {
+            table_name: @table_name,
+            item: json_obj
+        }
+
+    dynamodb.client.put_item(params)
+
+  end
+
+  def delete(key)
+
+    params =
+        {
+            table_name: @table_name,
+            key: key
+        }
+
+    dynamodb.client.delete_item(params)
+
+  end
+
+  def get_by_key(hash_key, hash_value, range_key = nil, range_value = nil)
+
+    key = {}
+    key[hash_key] = hash_value
+    if(range_key != nil)
+      key[range_key] = range_value
+    end
+
+    params = {
+        table_name: table_name,
+        key: key
+    }
+
+    result = dynamodb.client.get_item(params)
+    return map(result.item)
+
+  end
+
+  def all
+
+    result = dynamodb.client.scan({
+                                      :table_name => table_name
+                                  })
+
+    output = []
+    result.items.each do |item|
+      output.push(map(item))
+    end
+
+    return output
+
+  end
+
+  def scan(expression, expression_params, limit = nil, count = false)
+
+    params = {
+        :table_name => table_name
+    }
+
+    if expression != nil
+      params[:filter_expression] = expression
+    end
+
+    if expression_params != nil
+
+      params[:expression_attribute_names] = {}
+      params[:expression_attribute_values] = {}
+
+      expression_params.each do |key, value|
+        if key.starts_with?('#')
+          params[:expression_attribute_names][key] = value
+        elsif key.starts_with?(':')
+          params[:expression_attribute_values][key] = value
+        end
+      end
+
+    end
+
+    if limit != nil
+      params[:limit] = limit
+    end
+
+    if count
+      params[:select] = 'COUNT'
+    else
+      params[:select] = 'ALL_ATTRIBUTES'
+    end
+
+    result = dynamodb.client.scan(params)
+
+    if count
+      return result.count
+    else
+      output = []
+      result.items.each do |item|
+        output.push(map(item))
+      end
+
+      return output
+    end
+
+  end
+
+  def query(hash_key_name, hash_key_value, range_key_name = nil, range_key_value = nil, expression = nil, expression_params = nil, index_name = nil, limit = nil, count = false)
+
+    params = {
+        table_name: table_name
+    }
+
+    if expression != nil
+      params[:filter_expression] = expression
+    end
+
+    if index_name != nil
+      params[:index_name] = index_name
+    end
+
+    if range_key_name != nil
+      params[:key_condition_expression] = '#hash_key = :hash_key and #range_key = :range_key'
+      params[:expression_attribute_names] = { '#hash_key' => hash_key_name, '#range_key' => range_key_name }
+      params[:expression_attribute_values] = { ':hash_key' => hash_key_value, ':range_key' => range_key_value }
+    else
+      params[:key_condition_expression] = '#hash_key = :hash_key'
+      params[:expression_attribute_names] = { '#hash_key' => hash_key_name }
+      params[:expression_attribute_values] = { ':hash_key' => hash_key_value }
+    end
+
+    if expression_params != nil
+
+      expression_params.each do |key, value|
+        if key.starts_with?('#')
+          params[:expression_attribute_names][key] = value
+        elsif key.starts_with?(':')
+          params[:expression_attribute_values][key] = value
+        end
+      end
+
+    end
+
+    if limit != nil
+      params[:limit] = limit
+    end
+
+    if count
+      params[:select] = 'COUNT'
+    else
+      params[:select] = 'ALL_ATTRIBUTES'
+    end
+
+    result = dynamodb.client.query(params)
+
+    if count
+      return result.count
+    else
+      output = []
+      result.items.each do |item|
+        output.push(map(item))
+      end
+
+      return output
+    end
+
+  end
+
+  def map(item)
+    return item
+  end
+
+end

data/lib/dynamodb_framework/dynamodb_store.rb

@@ -0,0 +1,16 @@
+class DynamoDbStore
+  attr_reader :client
+
+  def initialize
+    region = ENV['DYNAMODB_REGION']
+    endpoint = ENV['DYNAMODB_ENDPOINT']
+    secret = ENV['DYNAMODB_SECRET']
+    key = ENV['DYNAMODB_ACCESS_KEY']
+    if endpoint != nil
+      @client = Aws::DynamoDB::Client.new(region: region, endpoint: endpoint, secret_access_key: secret, access_key_id: key)
+    else
+      @client = Aws::DynamoDB::Client.new(region: region, secret_access_key: secret, access_key_id: key)
+    end
+  end
+
+end

data/lib/dynamodb_framework/dynamodb_table_manager.rb

@@ -0,0 +1,283 @@
+class DynamoDbTableManager
+
+  attr_reader :dynamodb
+
+  def initialize
+    @dynamodb = DynamoDbStore.new
+  end
+
+  def exists?(table_name)
+
+    exists = true
+
+    begin
+      dynamodb.client.describe_table(:table_name => table_name)
+    rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+      exists = false
+    end
+
+    return exists
+
+  end
+
+  def has_index?(table_name, index_name)
+    exists = true
+
+    begin
+      result = dynamodb.client.describe_table(:table_name => table_name)
+
+      if result.table[:global_secondary_indexes] == nil
+        return false
+      end
+
+      if result.table[:global_secondary_indexes].select { |i| i[:index_name] == index_name }.length > 0
+        exists = true
+      else
+        exists = false
+      end
+    rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+      exists = false
+    end
+
+    return exists
+  end
+
+  def update_throughput(table_name, read_capacity, write_capacity)
+
+    if !exists?(table_name)
+      raise "table: #{table_name}, does not exist."
+    end
+
+    table = {
+        :table_name => table_name,
+        :provisioned_throughput => {
+            :read_capacity_units => read_capacity,
+            :write_capacity_units => write_capacity
+        }
+    }
+
+    dynamodb.client.update_table(table)
+
+    # wait for table to be updated
+    puts "waiting for table: [#{table_name}] to be updated..."
+    wait_until_active(table_name)
+    puts "table: [#{table_name}] updated!"
+
+  end
+
+  def add_index(table_name, attributes, global_index)
+
+    attribute_definitions = []
+
+    attributes.each do |a|
+      attribute_definitions.push({ :attribute_name => a[:name], :attribute_type => a[:type] })
+    end
+
+    table = {
+        :table_name => table_name,
+        :attribute_definitions => attribute_definitions,
+        :global_secondary_index_updates => [
+            :create => global_index
+        ]
+    }
+
+    dynamodb.client.update_table(table)
+
+    # wait for table to be updated
+    puts "Adding global index: #{global_index[:index_name]}..."
+    wait_until_index_active(table_name, global_index[:index_name])
+    puts "Index added!"
+  end
+
+  def update_index_throughput(table_name, index_name, read_capacity, write_capacity)
+    table = {
+        :table_name => table_name,
+        :global_secondary_index_updates => [
+            :update => {
+                :index_name => index_name,
+                :provisioned_throughput => {
+                    :read_capacity_units => read_capacity,
+                    :write_capacity_units => write_capacity
+                }
+            }
+        ]
+    }
+
+    dynamodb.client.update_table(table)
+
+    # wait for table to be updated
+    puts "Updating throughput for global index: #{index_name}..."
+    puts "waiting for table: [#{table_name}] to be updated..."
+    wait_until_active(table_name)
+    puts "table: [#{table_name}] updated!"
+  end
+
+  def drop_index(table_name, index_name)
+    table = {
+        :table_name => table_name,
+        :global_secondary_index_updates => [
+            :delete => {
+                :index_name => index_name
+            }
+        ]
+    }
+
+    dynamodb.client.update_table(table)
+
+    # wait for table to be updated
+    puts "Deleting global index: #{index_name}..."
+    wait_until_index_dropped(table_name, index_name)
+    puts "Index: [#{index_name}] dropped!"
+  end
+
+  def get_status(table_name)
+    result = dynamodb.client.describe_table(:table_name => table_name)
+    return result.table[:table_status]
+  end
+
+  def get_index_status(table_name, index_name)
+    result = dynamodb.client.describe_table(:table_name => table_name)
+
+    if result.table[:global_secondary_indexes] == nil
+      return nil
+    end
+
+    index = result.table[:global_secondary_indexes].select { |i| i[:index_name] == index_name }
+
+    if index.length > 0
+      return index[0][:index_status]
+    end
+
+    return nil
+  end
+
+  def wait_until_active(table_name)
+
+    end_time = Time.now + 5.minutes
+    while Time.now < end_time do
+
+      status = get_status(table_name)
+
+      if status == 'ACTIVE'
+        return
+      end
+
+      sleep(5.seconds)
+    end
+
+    raise "Timeout occured while waiting for table: #{table_name}, to become active."
+
+  end
+
+  def wait_until_index_active(table_name, index_name)
+
+    end_time = Time.now + 5.minutes
+    while Time.now < end_time do
+
+      status = get_index_status(table_name, index_name)
+
+      if status == 'ACTIVE'
+        return
+      end
+
+      sleep(5.seconds)
+    end
+
+    raise "Timeout occured while waiting for table: #{table_name}, index: #{index_name}, to become active."
+
+  end
+
+  def wait_until_index_dropped(table_name, index_name)
+
+    end_time = Time.now + 5.minutes
+    while Time.now < end_time do
+
+      status = get_index_status(table_name, index_name)
+
+      if status == nil
+        return
+      end
+
+      sleep(5.seconds)
+    end
+
+    raise "Timeout occured while waiting for table: #{table_name}, index: #{index_name}, to be dropped."
+
+  end
+
+  def create(table_name, attributes, partition_key, range_key = nil, read_capacity = 20, write_capacity = 10, global_indexes = nil)
+
+    if exists?(table_name)
+      return
+    end
+
+    attribute_definitions = []
+
+    attributes.each do |a|
+      attribute_definitions.push({ :attribute_name => a[:name], :attribute_type => a[:type] })
+    end
+
+    key_schema = []
+    key_schema.push({ :attribute_name => partition_key, :key_type => :HASH })
+    if range_key != nil
+      key_schema.push({ :attribute_name => range_key, :key_type => :RANGE })
+    end
+
+    table = {
+        :table_name => table_name,
+        :attribute_definitions => attribute_definitions,
+        :key_schema => key_schema,
+        :provisioned_throughput => {
+            :read_capacity_units => read_capacity,
+            :write_capacity_units => write_capacity
+        }
+    }
+
+    if global_indexes != nil
+      table[:global_secondary_indexes] = global_indexes
+    end
+
+    dynamodb.client.create_table(table)
+
+    # wait for table to be created
+    puts "waiting for table: [#{table_name}] to be created..."
+    dynamodb.client.wait_until(:table_exists, table_name: table_name)
+    puts "table: [#{table_name}] created!"
+  end
+
+  def create_global_index(name, partition_key, range_key = nil, read_capacity = 20, write_capacity = 10)
+
+    key_schema = []
+
+    key_schema.push({ :attribute_name => partition_key, :key_type => :HASH })
+    if range_key != nil
+      key_schema.push({ :attribute_name => range_key, :key_type => :RANGE })
+    end
+
+    index = {
+        :index_name => name,
+        :key_schema => key_schema,
+        :projection => {
+            :projection_type => :ALL
+        },
+        :provisioned_throughput => {
+            :read_capacity_units => read_capacity,
+            :write_capacity_units => write_capacity,
+        }
+    }
+
+    return index
+  end
+
+  def drop(table_name)
+
+    if !exists?(table_name)
+      return
+    end
+
+    puts "dropping table: [#{table_name}] ..."
+    dynamodb.client.delete_table({ table_name: table_name })
+    puts "table: [#{table_name}] dropped!"
+  end
+
+end

data/lib/dynamodb_framework/version.rb

@@ -0,0 +1,3 @@
+module DynamodbFramework
+  VERSION = "0.1.0"
+end