dynamo-record 0.2.0 → 0.3.0

data/docker-compose.yml

@@ -1,15 +1,20 @@
 version: '2'
 
 services:
-  test:
+  app:
     build:
       context: .
-      dockerfile: Dockerfile.test
+      dockerfile: Dockerfile
     environment:
       AWS_ACCESS_KEY_ID: x
       AWS_SECRET_ACCESS_KEY: x
       AWS_REGION: us-west-2
       DYNAMO_ENDPOINT: http://dynamo:8000
+      RAILS_ENV: test
+    logging:
+      options:
+        max-file: '1'
+        max-size: 5m
     links:
       - dynamo
 

data/dynamo-record.gemspec

@@ -1,38 +1,50 @@
-# coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'dynamo-record/record/version'
+require 'dynamo/record/version'
 
-Gem::Specification.new do |spec|
-  spec.name          = 'dynamo-record'
-  spec.version       = DynamoRecord::Record::VERSION
-  spec.license       = 'MIT'
-  spec.homepage      = 'https://instructure.com'
-  spec.authors       = ['Davis McClellan', 'Ryan Taylor', 'Bryan Petty', 'Michael Brewer-Davis', 'Marc Phillips',
-                        'Augusto Callejas', 'Frank Murphy']
-  spec.email         = ['dmcclellan@instructure.com', 'rtaylor@instructure.com', 'bpetty@instructure.com',
-                        'mbd@instructure.com', 'mphillips@instructure.com', 'acallejas@instructure.com',
-                        'fmurphy@instructure.com']
+Gem::Specification.new do |s|
+  s.name          = 'dynamo-record'
+  s.version       = Dynamo::Record::VERSION
+  s.summary       = 'Extensions to Aws::Record for working with DynamoDB.'
+  s.homepage      = 'https://github.com/instructure/dynamo-record'
+  s.license       = 'MIT'
 
-  spec.summary       = 'Extensions for working with dynamo via aws-record'
-  spec.description   = 'A set of extensions simplifying database operations in aws-record'
+  s.authors       = [
+    'Davis McClellan',
+    'Ryan Taylor',
+    'Bryan Petty',
+    'Michael Brewer-Davis',
+    'Marc Phillips',
+    'Augusto Callejas',
+    'Frank Murphy'
+  ]
+  s.email = [
+    'dmcclellan@instructure.com',
+    'rtaylor@instructure.com',
+    'bpetty@instructure.com',
+    'mbd@instructure.com',
+    'mphillips@instructure.com',
+    'acallejas@instructure.com',
+    'fmurphy@instructure.com'
+  ]
 
-  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+  s.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ['lib']
+  s.require_paths = ['lib']
 
-  spec.add_dependency 'aws-record', '~> 1.1'
-  spec.add_dependency 'rails', '~> 4.2'
+  s.add_dependency 'aws-record', '~> 1.1'
+  s.add_dependency 'activemodel', '>= 4.2', '< 5.2'
+  s.add_dependency 'railties', '>= 4.2', '< 5.2'
 
-  spec.add_development_dependency 'bundler', '~> 1.14'
-  spec.add_development_dependency 'byebug', '~> 9.0'
-  spec.add_development_dependency 'combustion', '~> 0.6.0'
-  spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.44.1'
-  spec.add_development_dependency 'simplecov', '~> 0.12'
-  spec.add_development_dependency 'webmock', '~> 2.1'
+  s.add_development_dependency 'activesupport', '>= 4.2', '< 5.2'
+  s.add_development_dependency 'bundler', '~> 1.15'
+  s.add_development_dependency 'byebug', '~> 9.0'
+  s.add_development_dependency 'combustion', '~> 0.6.0'
+  s.add_development_dependency 'rake', '~> 12.0'
+  s.add_development_dependency 'rspec', '~> 3.6'
+  s.add_development_dependency 'rubocop', '~> 0.50.0'
+  s.add_development_dependency 'simplecov', '~> 0.14'
+  s.add_development_dependency 'webmock', '~> 2.1'
+  s.add_development_dependency 'wwtd', '~> 1.3'
 end

data/lib/dynamo/record.rb

@@ -0,0 +1,17 @@
+require 'active_model'
+require 'aws-record'
+require 'rails/railtie'
+
+require 'dynamo/record/marshalers'
+require 'dynamo/record/model'
+require 'dynamo/record/railtie'
+require 'dynamo/record/model_existence_validator'
+require 'dynamo/record/table_migration'
+require 'dynamo/record/version'
+
+require 'dynamo/record/task_helpers/cleanup'
+require 'dynamo/record/task_helpers/drop_all_tables'
+require 'dynamo/record/task_helpers/drop_table'
+require 'dynamo/record/task_helpers/list_tables'
+require 'dynamo/record/task_helpers/migration_runner'
+require 'dynamo/record/task_helpers/scale'

data/lib/dynamo/record/marshalers.rb

@@ -0,0 +1,46 @@
+module Dynamo
+  module Record
+    module Marshalers
+      COMPOSITE_DELIMETER = '|'.freeze
+
+      def self.included(sub_class)
+        sub_class.extend(ClassMethods)
+        super(sub_class)
+      end
+
+      module ClassMethods
+        def composite_integer_attr(name, opts = {})
+          composite_attr(name, opts)
+          define_readers(name, opts[:parts], :to_i) if opts.key? :parts
+        end
+
+        def composite_string_attr(name, opts = {})
+          composite_attr(name, opts)
+          define_readers(name, opts[:parts], :to_s) if opts.key? :parts
+        end
+
+        private
+
+        def composite_attr(name, opts = {})
+          opts[:dynamodb_type] = 'S'
+
+          # It is very unfortunate that Aws::Record used `attr`
+          # rubocop:disable Style/Attr
+          attr(name, Aws::Record::Marshalers::StringMarshaler.new(opts), opts)
+          # rubocop:enable Style/Attr
+        end
+
+        def define_readers(name, parts, cast_function)
+          parts.each_with_index do |part, i|
+            raise "#{part} already defined" unless parts.find_index(part) == i
+            next if method_defined?(part)
+            define_method(part) do
+              # @data is used internally by Aws::Record to store all of the attributes
+              @data.get_attribute(name).split(COMPOSITE_DELIMETER)[i].send(cast_function)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamo/record/model.rb

@@ -0,0 +1,127 @@
+module Dynamo
+  module Record
+    module Model
+      COMPOSITE_DELIMITER = '|'.freeze
+
+      def self.included(klass)
+        klass.include(Aws::Record)
+        klass.include(Dynamo::Record::Marshalers)
+
+        klass.extend ClassMethods
+        klass.send :prepend, InstanceMethods
+      end
+
+      module ClassMethods
+        def table_name
+          [Rails.configuration.dynamo['prefix'], name.tableize].join('-').tr('/', '.')
+        end
+
+        def scan
+          raise 'no scanning in production' if Rails.env.production?
+          super
+        end
+
+        def find(opts)
+          super(opts).tap do |record|
+            unless record
+              name = self.name.demodulize
+              conditions = opts.map { |k, v| "#{k}=#{v}" }.join(', ')
+              error = "Couldn't find #{name} with #{conditions}"
+              raise Aws::Record::Errors::NotFound, error
+            end
+          end
+        end
+
+        def find_all_by_hash_key(hash_key_value, opts = {})
+          find_all_by_index_and_hash_key(hash_key, hash_key_value, opts)
+        end
+
+        def find_all_by_gsi_hash_key(gsi_name, hash_key_value, opts = {})
+          hash_key_name = global_secondary_indexes[gsi_name.to_sym][:hash_key]
+          find_all_by_index_and_hash_key(hash_key_name, hash_key_value, opts, gsi_name.to_s)
+        end
+
+        def find_all_by_gsi_hash_and_range_keys(gsi_name, hash_key_value, range_key_value)
+          gsi_config = global_secondary_indexes[gsi_name.to_sym]
+          find_all_by_index_hash_and_range_keys(
+            hash_config: { name: gsi_config[:hash_key], value: hash_key_value },
+            range_config: { name: gsi_config[:range_key], value: range_key_value },
+            index_name: gsi_name.to_s
+          )
+        end
+
+        def find_all_by_lsi_hash_key(lsi_name, hash_key_value, opts = {})
+          hash_key_name = local_secondary_indexes[lsi_name.to_sym][:hash_key]
+          find_all_by_index_and_hash_key(hash_key_name, hash_key_value, opts, lsi_name.to_s)
+        end
+
+        def find_all_by_lsi_hash_and_range_keys(lsi_name, hash_key_value, range_key_value)
+          lsi_config = local_secondary_indexes[lsi_name.to_sym]
+          find_all_by_index_hash_and_range_keys(
+            hash_config: { name: lsi_config[:hash_key], value: hash_key_value },
+            range_config: { name: lsi_config[:range_key], value: range_key_value },
+            index_name: lsi_name.to_s
+          )
+        end
+
+        def find_all_by_index_and_hash_key(hash_key_name, hash_key_value, opts = {}, index_name = nil)
+          query_options = {
+            select: 'ALL_ATTRIBUTES',
+            key_condition_expression: "#{hash_key_name} = :hash_key_value",
+            expression_attribute_values: {
+              ':hash_key_value': hash_key_value
+            },
+            scan_index_forward: true
+          }
+          query_options[:index_name] = index_name if index_name
+          query_options.merge!(opts)
+          query(query_options)
+        end
+
+        def find_all_by_index_hash_and_range_keys(hash_config:, range_config:, index_name: nil,
+          scan_index_forward: true, limit: nil)
+          range_expression = range_config[:expression] || "#{range_config[:name]} = :rkv"
+          query_options = {
+            select: 'ALL_ATTRIBUTES',
+            key_condition_expression: "#{hash_config[:name]} = :hkv AND #{range_expression}",
+            expression_attribute_values: {
+              ':hkv': hash_config[:value],
+              ':rkv': range_config[:value]
+            },
+            scan_index_forward: scan_index_forward,
+            limit: limit
+          }
+          query_options[:index_name] = index_name if index_name
+          query(query_options)
+        end
+
+        def composite_key(*args)
+          args.join(COMPOSITE_DELIMITER)
+        end
+
+        def split_composite(string)
+          string.split(COMPOSITE_DELIMITER)
+        end
+
+        # TODO: Create a batch save method.
+      end
+
+      module InstanceMethods
+        def read_attribute_for_serialization(attribute)
+          send(attribute)
+        end
+
+        def attribute_hash
+          attrs = self.class.attributes.attributes
+          attr_keys = attrs.keys
+          hash = {}
+
+          attr_keys.each do |key|
+            hash[key] = attrs[key].type_cast(send(key))
+          end
+          hash
+        end
+      end
+    end
+  end
+end

data/lib/dynamo/record/model_existence_validator.rb

@@ -0,0 +1,10 @@
+module Dynamo
+  module Record
+    class ModelExistenceValidator < ActiveModel::EachValidator
+      def validate_each(record, attribute, value)
+        return if options[:model].exists? value
+        record.errors[attribute] << (options[:message] || "#{attribute}:#{value} is not a valid #{options[:model]}")
+      end
+    end
+  end
+end

data/lib/{dynamo-record → dynamo}/record/railtie.rb

@@ -1,6 +1,4 @@
-require 'dynamo-record/record'
-require 'rails'
-module DynamoRecord
+module Dynamo
   module Record
     class Railtie < Rails::Railtie
       railtie_name :dynamo_record

data/lib/dynamo/record/table_migration.rb

@@ -0,0 +1,59 @@
+module Dynamo
+  module Record
+    class TableMigration
+      def self.table_config_check
+        if migrate_table?
+          table_config.migrate!
+          return :migrated
+        end
+        :exists
+      end
+
+      def self.migrate(model)
+        migration = Aws::Record::TableMigration.new(model)
+        begin
+          migration.client.describe_table(table_name: model.table_name)
+          return :exists
+        rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+          yield migration
+          migration.wait_until_available
+          return :migrated
+        end
+      end
+
+      def self.migrate_updates(model)
+        migration = Aws::Record::TableMigration.new(model)
+        yield migration
+        migration.wait_until_available
+        :updated
+      end
+
+      def self.add_stream(model)
+        migrate_updates(model) do |migration|
+          migration.update!(
+            stream_specification: {
+              stream_enabled: true,
+              stream_view_type: 'NEW_IMAGE'
+            }
+          )
+        end
+      rescue Aws::DynamoDB::Errors::ValidationException => e
+        return e.message if e.message == 'Table already has an enabled stream'
+        raise e
+      end
+
+      def self.migrate_table?(update_provisioned_throughput = false)
+        unless update_provisioned_throughput
+          table_name = table_config.instance_values['model_class'].table_name
+          described_table = table_config.client.describe_table table_name: table_name
+          provisioned_throughput = described_table.table.provisioned_throughput
+          table_config.read_capacity_units provisioned_throughput.read_capacity_units
+          table_config.write_capacity_units provisioned_throughput.write_capacity_units
+        end
+        !table_config.exact_match?
+      rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+        true
+      end
+    end
+  end
+end

data/lib/dynamo/record/task_helpers/cleanup.rb

@@ -0,0 +1,21 @@
+module Dynamo
+  module Record
+    module TaskHelpers
+      class Cleanup
+        def self.run
+          raise 'Task not available on production' if Rails.env.production?
+          Dir[Rails.root.join('app', 'models', '*.rb').to_s].each do |filename|
+            delete_by_class(filename)
+          end
+        end
+
+        def self.delete_by_class(filename)
+          klass = File.basename(filename, '.rb').camelize.constantize
+          return unless klass.included_modules.include? Dynamo::Record::Model
+          Rails.logger.info "Deleting all items in table: #{klass}"
+          klass.scan.each(&:delete!)
+        end
+      end
+    end
+  end
+end

data/lib/dynamo/record/task_helpers/drop_all_tables.rb

@@ -0,0 +1,19 @@
+module Dynamo
+  module Record
+    module TaskHelpers
+      class DropAllTables
+        def self.run(override = false)
+          raise 'Task not available on production' if Rails.env.production?
+          env = Rails.env
+          dynamodb = Aws::DynamoDB::Client.new
+          tables = dynamodb.list_tables
+          tables.table_names.map do |t|
+            next unless t.include?(env) || override
+            dt = dynamodb.delete_table(table_name: t)
+            dt ? "Deleted: #{t}" : "Delete failed: #{t}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamo/record/task_helpers/drop_table.rb

@@ -0,0 +1,13 @@
+module Dynamo
+  module Record
+    module TaskHelpers
+      class DropTable
+        def self.run(table_name)
+          dynamodb = Aws::DynamoDB::Client.new
+          resp = dynamodb.delete_table(table_name: table_name)
+          "Deleted: #{resp.table_description.table_name}"
+        end
+      end
+    end
+  end
+end

data/lib/dynamo/record/task_helpers/list_tables.rb

@@ -0,0 +1,15 @@
+module Dynamo
+  module Record
+    module TaskHelpers
+      class ListTables
+        def self.run
+          dynamodb = Aws::DynamoDB::Client.new
+          tables = dynamodb.list_tables
+          tables.table_names.select do |tn|
+            tn.starts_with?(Rails.configuration.dynamo['prefix'])
+          end
+        end
+      end
+    end
+  end
+end