dynamo-record 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 77cdbe44e4be22d2294565970b108d66c48ffb87
+  data.tar.gz: 70653d3d5cddc536eeefb74d47ea8522665f4b3b
+SHA512:
+  metadata.gz: b7fe3b6bd23079f734531545ee1ae42079d5053b99b058d9cc1c96a0ba78804de27b8f3fca6ed7609d299d7c20bf0bfd21c38b34823099b8da6cf254197c2537
+  data.tar.gz: ec96773a915b361673ee1af591ea13a84b8cb5724cc5fc76168412d75cb43bd590d7b721462c2a682a677237459bc0449ee0c5e8f660806aa33097e3157c3731

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,81 @@
+Rails:
+  Enabled: true
+
+Rails/HttpPositionalArguments:
+  # Renable once we are on Rails v5
+  Enabled: false
+
+AllCops:
+  TargetRubyVersion: 2.3
+
+Metrics/ClassLength:
+  Max: 200 # Default: 100
+
+Metrics/LineLength:
+  Max: 120 # Default: 80
+
+Metrics/MethodLength:
+  Max: 20 # Default: 10
+
+Metrics/BlockLength:
+  Max: 30
+  Exclude:
+    - 'spec/**/*.rb'
+
+Style/AlignParameters:
+  # Alignment of parameters in multi-line method calls.
+  #
+  # The `with_fixed_indentation` style aligns the following lines with one
+  # level of indentation relative to the start of the line with the method call.
+  #
+  #     method_call(a,
+  #       b)
+  EnforcedStyle: with_fixed_indentation
+
+Style/ClassAndModuleChildren:
+  # Checks the style of children definitions at classes and modules.
+  #
+  # Basically there are two different styles:
+  #
+  # `nested` - have each child on a separate line
+  #   class Foo
+  #     class Bar
+  #     end
+  #   end
+  #
+  # `compact` - combine definitions as much as possible
+  #   class Foo::Bar
+  #   end
+  #
+  # The compact style is only forced, for classes / modules with one child.
+  EnforcedStyle: nested
+  Enabled: false
+
+Style/Documentation:
+  # This cop checks for missing top-level documentation of classes and modules.
+  # Classes with no body and namespace modules are exempt from the check.
+  # Namespace modules are modules that have nothing in their bodies except
+  # classes or other modules.
+  Enabled: false
+
+Lint/EndAlignment:
+  AlignWith: variable
+
+Style/CaseIndentation:
+  IndentWhenRelativeTo: end
+
+Style/FrozenStringLiteralComment:
+  # `when_needed` will add the frozen string literal comment to files
+  # only when the `TargetRubyVersion` is set to 2.3+.
+  # `always` will always add the frozen string literal comment to a file
+  # regardless of the Ruby version or if `freeze` or `<<` are called on a
+  # string literal. If you run code against multiple versions of Ruby, it is
+  # possible that this will create errors in Ruby 2.3.0+.
+  #
+  # See: https://wyeworks.com/blog/2015/12/1/immutable-strings-in-ruby-2-dot-3
+  EnforcedStyle: when_needed
+  Enabled: false
+
+Style/NumericPredicate:
+  Exclude:
+    - 'spec/**/*'

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.14.6

data/Dockerfile.test

@@ -0,0 +1,23 @@
+FROM instructure/ruby:2.3
+
+ENV APP_HOME "/usr/src/app/"
+
+USER root
+
+COPY dynamo-record.gemspec Gemfile Gemfile.lock $APP_HOME
+RUN mkdir -p $APP_HOME/lib/dynamo-record/record
+COPY lib/dynamo-record/record/version.rb $APP_HOME/lib/dynamo-record/record
+RUN chown -R docker:docker $APP_HOME
+
+USER docker
+RUN gem install bundler
+RUN bundle install --quiet --jobs 8
+USER root
+
+COPY . $APP_HOME
+RUN mkdir -p $APP_HOME/coverage && \
+    mkdir -p $APP_HOME/spec/internal/log && \
+    chown -R docker:docker $APP_HOME
+USER docker
+
+CMD ["bundle", "exec", "rspec"]

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in dynamo-record.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,163 @@
+PATH
+  remote: .
+  specs:
+    dynamo-record (0.1.0)
+      aws-record (~> 1.1)
+      rails (~> 4.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (4.2.8)
+      actionpack (= 4.2.8)
+      actionview (= 4.2.8)
+      activejob (= 4.2.8)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+    actionpack (4.2.8)
+      actionview (= 4.2.8)
+      activesupport (= 4.2.8)
+      rack (~> 1.6)
+      rack-test (~> 0.6.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (4.2.8)
+      activesupport (= 4.2.8)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (4.2.8)
+      activesupport (= 4.2.8)
+      globalid (>= 0.3.0)
+    activemodel (4.2.8)
+      activesupport (= 4.2.8)
+      builder (~> 3.1)
+    activerecord (4.2.8)
+      activemodel (= 4.2.8)
+      activesupport (= 4.2.8)
+      arel (~> 6.0)
+    activesupport (4.2.8)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.5.1)
+      public_suffix (~> 2.0, >= 2.0.2)
+    arel (6.0.4)
+    aws-record (1.1.0)
+      aws-sdk-resources (~> 2.0)
+    aws-sdk-core (2.9.11)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.9.11)
+      aws-sdk-core (= 2.9.11)
+    aws-sigv4 (1.0.0)
+    builder (3.2.3)
+    byebug (9.0.6)
+    combustion (0.6.0)
+      activesupport (>= 3.0.0)
+      railties (>= 3.0.0)
+      thor (>= 0.14.6)
+    concurrent-ruby (1.0.5)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    docile (1.1.5)
+    erubis (2.7.0)
+    globalid (0.4.0)
+      activesupport (>= 4.2.0)
+    hashdiff (0.3.2)
+    i18n (0.8.1)
+    jmespath (1.3.1)
+    json (2.1.0)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    mail (2.6.4)
+      mime-types (>= 1.16, < 4)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.1.0)
+    minitest (5.10.1)
+    nokogiri (1.7.1)
+      mini_portile2 (~> 2.1.0)
+    public_suffix (2.0.5)
+    rack (1.6.5)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails (4.2.8)
+      actionmailer (= 4.2.8)
+      actionpack (= 4.2.8)
+      actionview (= 4.2.8)
+      activejob (= 4.2.8)
+      activemodel (= 4.2.8)
+      activerecord (= 4.2.8)
+      activesupport (= 4.2.8)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.2.8)
+      sprockets-rails
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.8)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    railties (4.2.8)
+      actionpack (= 4.2.8)
+      activesupport (= 4.2.8)
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (10.5.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    safe_yaml (1.0.4)
+    simplecov (0.14.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    sprockets (3.7.1)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.0)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    thor (0.19.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    webmock (2.3.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.14)
+  byebug (~> 9.0)
+  combustion (~> 0.6.0)
+  dynamo-record!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  simplecov (~> 0.12)
+  webmock (~> 2.1)
+
+BUNDLED WITH
+   1.14.6

data/README.md

@@ -0,0 +1,110 @@
+# DynamoRecord
+
+Provides helpful rake tasks and model extensions on top of [aws-record](https://github.com/aws/aws-sdk-ruby-record).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'dynamo-record'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dynamo-record
+
+## Usage
+
+### Models
+
+In `app/models`, create a class that includes `DynamoRecord::Model` and contains one or more of the following attribute declarations:
+
+* `integer_attr`
+* `float_attr`
+* `map_attr`
+* `composite_string_attr`
+* `composite_integer_attr`
+
+An example file:
+
+```
+class Model1
+  include DynamoRecord::Model
+  composite_string_attr(
+    :model_key,
+    hash_key: true,
+    parts: [:model_id, :account_id]
+  )
+  integer_attr :position, range_key: true
+  string_attr  :user_id
+  float_attr   :score
+  map_attr     :map_value
+  composite_string_attr :quiz_key, parts: [:quiz_id]
+end
+```
+
+The partition key is labeled with `hash_key: true`. The sort key is labeled with `range_key: true`.
+
+A secondary index can be defined as follows:
+
+```
+  global_secondary_index(
+    :secondary_idx,
+    hash_key: :user_id,
+    range_key: :score,
+    projection: {
+      projection_type: 'INCLUDE',
+      non_key_attributes: [
+        :map_value
+      ]
+    }
+  )
+```
+
+The full documentation for `projection_type` and `non_key_attributes` can be found [here](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-dynamodb-projectionobject.html).
+
+### Migrations
+
+Dynamo migration files are stored in `db/dynamo_migrate`. The name of the file follows the style of standard Rails migration files like `YYYYMMDDHHMMSS_create_model_1.rb`. A migration file that creates a Dynamo table looks like:
+
+```
+module DynamoMigrate
+  class CreateModel1 < DynamoRecord::TableMigration
+    def self.up
+      migrate(Model1) do |migration|
+        migration.create!(
+          provisioned_throughput: {
+            read_capacity_units: 1,
+            write_capacity_units: 1
+          },
+          global_secondary_index_throughput: {
+            secondary_idx: {
+              read_capacity_units: 1,
+              write_capacity_units: 1
+            }
+          }
+        )
+      end
+    end
+  end
+end
+```
+
+Note that the throughput of the table can be configured separately from the throughput of the secondary index.
+
+A migration file that creates a Dynamo stream looks like:
+
+```
+module DynamoMigrate
+  class AddModel1Stream < DynamoRecord::TableMigration
+    def self.update
+      add_stream(Model1)
+    end
+  end
+end
+```

data/build.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+export COMPOSE_PROJECT_NAME=dynamorecord
+export COMPOSE_FILE=docker-compose.yml
+
+function cleanup() {
+  exit_code=$?
+  set +e
+  docker-compose stop test
+  docker-compose rm -fa test
+  docker rmi -f $(docker images -qf "dangling=true") &>/dev/null
+  exit $exit_code
+}
+trap cleanup INT TERM EXIT
+
+set -e
+
+docker-compose build --pull
+docker-compose run --name test test
+docker cp test:/usr/src/app/coverage .
+
+#echo "Reporting test coverage"
+#if [[ $? -eq 0 && -a 'coverage/.last_run.json' ]]; then
+#  ruby_coverage=$(cat coverage/api/.last_run.json | grep 'covered_percent' | egrep -o '[0-9]+(\.[0-9]+)?')
+#  echo "Sending to gergich"
+#  gergich message ":ruby: <http://jenkins.instructure.com/job/dynamo-record/${BUILD_ID}/|${ruby_coverage}%>"
+#  gergich publish
+#fi

data/docker-compose.override.yml

@@ -0,0 +1,6 @@
+version: '2'
+
+services:
+  test:
+    volumes:
+      - .:/usr/src/app

data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2'
+
+services:
+  test:
+    build:
+      context: .
+      dockerfile: Dockerfile.test
+    environment:
+      AWS_ACCESS_KEY_ID: x
+      AWS_SECRET_ACCESS_KEY: x
+      AWS_REGION: us-west-2
+      DYNAMO_ENDPOINT: http://dynamo:8000
+    links:
+      - dynamo
+
+  dynamo:
+    image: instructure/dynamodb

data/dynamo-record.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dynamo-record/record/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'dynamo-record'
+  spec.version       = DynamoRecord::Record::VERSION
+  spec.license       = 'MIT'
+  spec.homepage      = 'https://instructure.com'
+  spec.authors       = ['Davis McClellan', 'Ryan Taylor', 'Bryan Petty', 'Michael Brewer-Davis', 'Marc Phillips', 'Augusto Callejas']
+  spec.email         = ['dmcclellan@instructure.com', 'rtaylor@instructure.com', 'bpetty@instructure.com',
+                        'mbd@instructure.com', 'mphillips@instructure.com', 'acallejas@instructure.com']
+
+  spec.summary       = 'Extensions for working with dynamo via aws-record'
+  spec.description   = 'A set of extensions simplifying database operations in aws-record'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'aws-record', '~> 1.1'
+  spec.add_dependency 'rails', '~> 4.2'
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'byebug', '~> 9.0'
+  spec.add_development_dependency 'combustion', '~> 0.6.0'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'simplecov', '~> 0.12'
+  spec.add_development_dependency 'webmock', '~> 2.1'
+end

data/lib/dynamo-record.rb

@@ -0,0 +1,4 @@
+Gem.find_files("#{File.dirname(__FILE__)}/dynamo-record/**/*.rb").each do |path|
+  require path.gsub(/\.rb$/, '')
+end
+require 'model_existence_validator'

data/lib/dynamo-record/marshalers.rb

@@ -0,0 +1,44 @@
+module DynamoRecord
+  module Marshalers
+    COMPOSITE_DELIMETER = '|'.freeze
+
+    def self.included(sub_class)
+      sub_class.extend(ClassMethods)
+      super(sub_class)
+    end
+
+    module ClassMethods
+      def composite_integer_attr(name, opts = {})
+        composite_attr(name, opts)
+        define_readers(name, opts[:parts], :to_i) if opts.key? :parts
+      end
+
+      def composite_string_attr(name, opts = {})
+        composite_attr(name, opts)
+        define_readers(name, opts[:parts], :to_s) if opts.key? :parts
+      end
+
+      private
+
+      def composite_attr(name, opts = {})
+        opts[:dynamodb_type] = 'S'
+
+        # It is very unfortunate that Aws::Record used `attr`
+        # rubocop:disable Style/Attr
+        attr(name, Aws::Record::Marshalers::StringMarshaler.new(opts), opts)
+        # rubocop:enable Style/Attr
+      end
+
+      def define_readers(name, parts, cast_function, allowed_repeats = [:account_uuid])
+        parts.each_with_index do |part, i|
+          raise "#{part} already defined" unless parts.find_index(part) == i
+          next if method_defined?(part)
+          define_method(part) do
+            # @data is used internally by Aws::Record to store all of the attributes
+            @data.get_attribute(name).split(COMPOSITE_DELIMETER)[i].send(cast_function)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamo-record/model.rb

@@ -0,0 +1,122 @@
+require 'aws-record'
+
+module DynamoRecord::Model
+  COMPOSITE_DELIMITER = '|'.freeze
+
+  def self.included(klass)
+    klass.include(Aws::Record)
+    klass.include(DynamoRecord::Marshalers)
+
+    klass.extend ClassMethods
+    klass.send :prepend, InstanceMethods
+  end
+
+  module ClassMethods
+    def table_name
+      [Rails.configuration.dynamo['prefix'], name.tableize].join('-').tr('/', '.')
+    end
+
+    def scan
+      raise 'no scanning in production' if Rails.env.production?
+      super
+    end
+
+    def find(opts)
+      super(opts).tap do |record|
+        unless record
+          name = self.name.demodulize
+          conditions = opts.map { |k, v| "#{k}=#{v}" }.join(', ')
+          error = "Couldn't find #{name} with #{conditions}"
+          raise Aws::Record::Errors::NotFound, error
+        end
+      end
+    end
+
+    def find_all_by_hash_key(hash_key_value, opts = {})
+      find_all_by_index_and_hash_key(hash_key, hash_key_value, opts)
+    end
+
+    def find_all_by_gsi_hash_key(gsi_name, hash_key_value, opts = {})
+      hash_key_name = global_secondary_indexes[gsi_name.to_sym][:hash_key]
+      find_all_by_index_and_hash_key(hash_key_name, hash_key_value, opts, gsi_name.to_s)
+    end
+
+    def find_all_by_gsi_hash_and_range_keys(gsi_name, hash_key_value, range_key_value)
+      gsi_config = global_secondary_indexes[gsi_name.to_sym]
+      find_all_by_index_hash_and_range_keys(
+        hash_config: { name: gsi_config[:hash_key], value: hash_key_value },
+        range_config: { name: gsi_config[:range_key], value: range_key_value },
+        index_name: gsi_name.to_s
+      )
+    end
+
+    def find_all_by_index_and_hash_key(hash_key_name, hash_key_value, opts = {}, index_name = nil)
+      query_options = {
+        select: 'ALL_ATTRIBUTES',
+        key_condition_expression: "#{hash_key_name} = :hash_key_value",
+        expression_attribute_values: {
+          ':hash_key_value': hash_key_value
+        },
+        scan_index_forward: true
+      }
+      query_options[:index_name] = index_name if index_name
+      query_options.merge!(opts)
+      query(query_options)
+    end
+
+    def find_all_by_index_hash_and_range_keys(hash_config:, range_config:, index_name: nil,
+      scan_index_forward: true, limit: nil)
+      query_options = {
+        select: 'ALL_ATTRIBUTES',
+        key_condition_expression: "#{hash_config[:name]} = :hkv AND #{range_config[:name]} = :rkv",
+        expression_attribute_values: {
+          ':hkv': hash_config[:value],
+          ':rkv': range_config[:value]
+        },
+        scan_index_forward: scan_index_forward,
+        limit: limit
+      }
+      query_options[:index_name] = index_name if index_name
+      query(query_options)
+    end
+
+    def composite_key(*args)
+      args.join(COMPOSITE_DELIMITER)
+    end
+
+    def split_composite(string)
+      string.split(COMPOSITE_DELIMITER)
+    end
+
+    def find_all_by_model(model, opts = {})
+      find_all_by_hash_key(model.dynamo_key, opts)
+    end
+
+    def find_all_by_model_paginated(model, opts = {})
+      find_all_by_model(model, pagination_opts(opts))
+    end
+
+    def pagination_opts(opts = {})
+      { limit: opts[:limit], exclusive_start_key: opts[:exclusive_start_key] }
+    end
+
+    # TODO: Create a batch save method.
+  end
+
+  module InstanceMethods
+    def read_attribute_for_serialization(attribute)
+      send(attribute)
+    end
+
+    def attribute_hash
+      attrs = self.class.attributes.attributes
+      attr_keys = attrs.keys
+      hash = {}
+
+      attr_keys.each do |key|
+        hash[key] = attrs[key].type_cast(send(key))
+      end
+      hash
+    end
+  end
+end

data/lib/dynamo-record/record.rb

@@ -0,0 +1,7 @@
+require 'dynamo-record/record/version'
+
+module DynamoRecord
+  module Record
+    require 'dynamo-record/record/railtie' if defined? Rails
+  end
+end

data/lib/dynamo-record/record/railtie.rb

@@ -0,0 +1,13 @@
+require 'dynamo-record/record'
+require 'rails'
+module DynamoRecord
+  module Record
+    class Railtie < Rails::Railtie
+      railtie_name :dynamo_record
+
+      rake_tasks do
+        load "tasks/dynamo.rake"
+      end
+    end
+  end
+end

data/lib/dynamo-record/record/version.rb

@@ -0,0 +1,5 @@
+module DynamoRecord
+  module Record
+    VERSION = '0.1.0'.freeze
+  end
+end

data/lib/dynamo-record/table_migration.rb

@@ -0,0 +1,36 @@
+module DynamoRecord
+  class TableMigration
+    def self.migrate(model)
+      migration = Aws::Record::TableMigration.new(model)
+      begin
+        migration.client.describe_table(table_name: model.table_name)
+        return :exists
+      rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+        yield migration
+        migration.wait_until_available
+        return :migrated
+      end
+    end
+
+    def self.migrate_updates(model)
+      migration = Aws::Record::TableMigration.new(model)
+      yield migration
+      migration.wait_until_available
+      :updated
+    end
+
+    def self.add_stream(model)
+      migrate_updates(model) do |migration|
+        migration.update!(
+          stream_specification: {
+            stream_enabled: true,
+            stream_view_type: 'NEW_IMAGE'
+          }
+        )
+      end
+    rescue Aws::DynamoDB::Errors::ValidationException => e
+      return e.message if e.message == 'Table already has an enabled stream'
+      raise e
+    end
+  end
+end

data/lib/dynamo-record/task_helpers/cleanup.rb

@@ -0,0 +1,16 @@
+module DynamoRecord
+  module TaskHelpers
+    class Cleanup
+      def self.run
+        raise 'Task not available on production' if Rails.env.production?
+        Dir[Rails.root.join('app/models/*.rb').to_s].each do |filename|
+          klass = File.basename(filename, '.rb').camelize.constantize
+          if klass.included_modules.include? DynamoRecord::Model
+            puts "Deleting all items in table: #{klass}"
+            klass.scan.each(&:delete!)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamo-record/task_helpers/drop_all_tables.rb

@@ -0,0 +1,17 @@
+module DynamoRecord
+  module TaskHelpers
+    class DropAllTables
+      def self.run(override = false)
+        raise 'Task not available on production' if Rails.env.production?
+        env = Rails.env
+        dynamodb = Aws::DynamoDB::Client.new
+        tables = dynamodb.list_tables
+        tables.table_names.map do |t|
+          next unless t.include?(env) || override
+          dt = dynamodb.delete_table(table_name: t)
+          dt ? "Deleted: #{t}" : "Delete failed: #{t}"
+        end
+      end
+    end
+  end
+end

data/lib/dynamo-record/task_helpers/drop_table.rb

@@ -0,0 +1,11 @@
+module DynamoRecord
+  module TaskHelpers
+    class DropTable
+      def self.run(table_name)
+        dynamodb = Aws::DynamoDB::Client.new
+        resp = dynamodb.delete_table(table_name: table_name)
+        "Deleted: #{resp.table_description.table_name}"
+      end
+    end
+  end
+end

data/lib/dynamo-record/task_helpers/list_tables.rb

@@ -0,0 +1,13 @@
+module DynamoRecord
+  module TaskHelpers
+    class ListTables
+      def self.run
+        dynamodb = Aws::DynamoDB::Client.new
+        tables = dynamodb.list_tables
+        tables.table_names.select do |tn|
+          tn.starts_with?(Rails.configuration.dynamo['prefix'])
+        end
+      end
+    end
+  end
+end

data/lib/dynamo-record/task_helpers/migration_runner.rb

@@ -0,0 +1,53 @@
+module DynamoRecord
+  module TaskHelpers
+    class MigrationRunner
+      def self.run(path='db/dynamo_migrate')
+        constants = []
+        filename_regexp = /\A([0-9]+)_([_a-z0-9]*)\.?([_a-z0-9]*)?\.rb\z/
+
+        # Sorts the files located in `db/dynamo_migrate` to ensure order is preserved
+        Dir[Rails.root.join("#{path}/*.rb")].sort.each do |f|
+          raise "Non-numeric prefix: #{f}" if File.basename(f).scan(filename_regexp).first.nil?
+          require f
+
+          # finds the constant that was added on the require statement above
+          migration_sym = (DynamoMigrate.constants - constants).first
+          migration = DynamoMigrate.const_get(migration_sym)
+          constants.push migration_sym
+
+          # starts the migration
+          yield "Migrating: #{migration}"
+
+          begin
+            status = up(migration)
+            yield status if status
+
+            status = update(migration)
+            yield status if status
+          rescue => e
+            yield "Migration failed: #{e}"
+          end
+        end
+      end
+
+      def self.up(migration)
+        return unless migration.respond_to? :up
+        case migration.up
+        when :exists
+          'Table already exists'
+        when :migrated
+          'Migration successful'
+        else
+          raise 'Migration failed'
+        end
+      end
+
+      def self.update(migration)
+        return unless migration.respond_to? :update
+        status = migration.update
+        return 'Migration successful' if status == :updated
+        status
+      end
+    end
+  end
+end

data/lib/dynamo-record/task_helpers/scale.rb

@@ -0,0 +1,86 @@
+module DynamoRecord
+  module TaskHelpers
+    class Scale
+      attr_reader :model, :attribute_selector, :new_throughput
+      attr_reader :migration, :existing_throughput, :model_name
+
+      def initialize(model_name, attribute_selector, new_throughput)
+        @model_name = model_name
+        @attribute_selector = attribute_selector
+        @new_throughput = new_throughput
+      end
+
+      def run
+        return description if [model_name, attribute_selector, new_throughput].any?(&:nil?)
+
+        @model = model_name.constantize
+        @migration = Aws::Record::TableMigration.new(model)
+        @existing_throughput = model.provisioned_throughput
+
+        update_throughput
+        success_message
+      end
+
+      private
+
+      def success_message
+        "Successfully updated #{model.table_name} throughput to #{update_instructions[:provisioned_throughput]}"
+      end
+
+      def update_throughput
+        raise_attribute_error if !read? && !write?
+
+        migration.update!(update_instructions)
+      end
+
+      def update_instructions
+        {
+          provisioned_throughput: {
+            write_capacity_units: (write? && new_throughput) || existing_write,
+            read_capacity_units: (read? && new_throughput) || existing_read
+          }
+        }
+      end
+
+      def existing_write
+        existing_throughput[:write_capacity_units]
+      end
+
+      def existing_read
+        existing_throughput[:read_capacity_units]
+      end
+
+      def both?
+        attribute_selector.to_sym == :both
+      end
+
+      def read?
+        both? || attribute_selector.to_sym == :read
+      end
+
+      def write?
+        both? || attribute_selector.to_sym == :write
+      end
+
+      def raise_attribute_error
+        raise ArgumentError, 'You didn\'t provide an appropriate attribute selection.  We  accept [:both, :read, :write]'
+      end
+
+      def description
+        <<~DESCRIPTION
+        --------------------------------------------------------------------------------
+          Here's some usage information:
+            Scale a dynamo table.  Requires three inputs.
+              - ModelName
+                ruby class of the model
+              - attribute
+                valid values include "both", "read" and "write"
+              - new_throughput
+                numerical value for the new read/write capacity units
+            Example: `rake dynamo:scale[MySuperDynamoModel,both,50]`
+          --------------------------------------------------------------------------------
+        DESCRIPTION
+      end
+    end
+  end
+end

data/lib/model_existence_validator.rb

@@ -0,0 +1,7 @@
+require 'active_model'
+class ModelExistenceValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    return if options[:model].exists? value
+    record.errors[attribute] << (options[:message] || "#{attribute}:#{value} is not a valid #{options[:model]}")
+  end
+end

data/lib/tasks/dynamo.rake

@@ -0,0 +1,38 @@
+namespace :dynamo do
+  desc 'Run dynamo migrations'
+  task migrate: :environment do
+    DynamoRecord::TaskHelpers::MigrationRunner.run { |msg| puts msg }
+  end
+
+  desc 'Drops all dynamo tables and re-runs migrations'
+  task :reset do
+    Rake::Task['dynamo:drop_all'].invoke
+    Rake::Task['dynamo:migrate'].invoke
+  end
+
+  desc 'Drop all dynamo tables'
+  task :drop_all, [:override] => :environment do |_t, args|
+    puts DynamoRecord::TaskHelpers::DropAllTables.run args[:override]
+  end
+
+  desc 'Drop a specified dynamo table'
+  task :drop, [:table_name] => :environment do |_t, args|
+    puts DynamoRecord::TaskHelpers::DropTable.run args[:table_name]
+  end
+
+  desc 'List all tables with dynamo prefix'
+  task list_tables: :environment do
+    puts DynamoRecord::TaskHelpers::ListTables.run
+  end
+
+  desc 'Delete all records in all DyanmoDB tables.'
+  task cleanup: :environment do
+    DynamoRecord::TaskHelpers::Cleanup.run
+    puts 'Finished deleting all records in all DynamoDB tables.'
+  end
+
+  desc 'Scale a dynamo table read/write capacity to the new capacity value provided'
+  task :scale, [:model_name, :attribute, :new_throughput] => :environment do |_t, args|
+    puts DynamoRecord::TaskHelpers::Scale.new(args[:model_name], args[:attribute], args[:new_throughput].to_i).run
+  end
+end

metadata

@@ -0,0 +1,207 @@
+--- !ruby/object:Gem::Specification
+name: dynamo-record
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Davis McClellan
+- Ryan Taylor
+- Bryan Petty
+- Michael Brewer-Davis
+- Marc Phillips
+- Augusto Callejas
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: combustion
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: A set of extensions simplifying database operations in aws-record
+email:
+- dmcclellan@instructure.com
+- rtaylor@instructure.com
+- bpetty@instructure.com
+- mbd@instructure.com
+- mphillips@instructure.com
+- acallejas@instructure.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Dockerfile.test
+- Gemfile
+- Gemfile.lock
+- README.md
+- build.sh
+- docker-compose.override.yml
+- docker-compose.yml
+- dynamo-record.gemspec
+- lib/dynamo-record.rb
+- lib/dynamo-record/marshalers.rb
+- lib/dynamo-record/model.rb
+- lib/dynamo-record/record.rb
+- lib/dynamo-record/record/railtie.rb
+- lib/dynamo-record/record/version.rb
+- lib/dynamo-record/table_migration.rb
+- lib/dynamo-record/task_helpers/cleanup.rb
+- lib/dynamo-record/task_helpers/drop_all_tables.rb
+- lib/dynamo-record/task_helpers/drop_table.rb
+- lib/dynamo-record/task_helpers/list_tables.rb
+- lib/dynamo-record/task_helpers/migration_runner.rb
+- lib/dynamo-record/task_helpers/scale.rb
+- lib/model_existence_validator.rb
+- lib/tasks/dynamo.rake
+homepage: https://instructure.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Extensions for working with dynamo via aws-record
+test_files: []