dynamo-record 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40c3a218185cf95ac6577f5e2954740ce8691bd4
-  data.tar.gz: d129868fe0ddc2eee7487dec4e59f4252ad7d0f7
+  metadata.gz: e0e65c5e7c2ecaac423640b5872a8c40d58a539a
+  data.tar.gz: 807766fdecdccdd5029a11a1f0fcbad8a2e61569
 SHA512:
-  metadata.gz: 27c3831739a5c405d87100ac1a8c4bbf4292e349ff0b24c6e1215fe19360867cb58917d6db92f7ff58175e20adcf7330c794748a2fbda8b90eaf85caa31b45bb
-  data.tar.gz: 697e619b45192d0e22e1afc77707014541a41680a51032e833dc7c88e9e65b728740fa88b53263416d637a69fd7febfac52dd146f63fb3381e2a96f564f1b704
+  metadata.gz: 5213186128c25aca4a3a37a27ee8534f7054f9bac3e9908c930a857d43811af1628a66f5ff76a551190a3144af6d98c464678c8becf6859d8cd8fa61e78d67d7
+  data.tar.gz: efefa9924786b3068896273ebbe3711d1eb2a7872e5270b15e735e81fc8cf9051b304f1fd3d036c17a7707cf252bdf6717bb48416c81f0c03025204c46aefc48

data/lib/dynamo/record.rb

@@ -3,6 +3,7 @@ require 'aws-record'
 require 'rails/railtie'
 
 require 'dynamo/record/marshalers'
+require 'dynamo/record/batch_get'
 require 'dynamo/record/batch_write'
 require 'dynamo/record/batch_request'
 require 'dynamo/record/model'
@@ -17,3 +18,15 @@ require 'dynamo/record/task_helpers/drop_table'
 require 'dynamo/record/task_helpers/list_tables'
 require 'dynamo/record/task_helpers/migration_runner'
 require 'dynamo/record/task_helpers/scale'
+
+module Dynamo
+  module Record
+    class << self
+      attr_writer :logger
+
+      def logger
+        @logger ||= Logger.new($stdout).tap { |log| log.progname = self.name }
+      end
+    end
+  end
+end

data/lib/dynamo/record/batch_get.rb

@@ -0,0 +1,81 @@
+require 'json'
+
+module Dynamo
+  module Record
+    module BatchGet
+      BATCH_SIZE = 100
+
+      module ClassMethods
+        def batch_get(keys)
+          process_batch_get_requests(keys)
+        end
+
+        private
+
+        def process_batch_get_requests(keys)
+          attempt = 1
+          results = []
+          unprocessed_keys = keys.dup.map(&:stringify_keys)
+          while !unprocessed_keys.empty? && attempt <= max_retries
+            batch = unprocessed_keys.shift(BATCH_SIZE)
+            Dynamo::Record.logger.debug "#{name} batch_get_item #{batch.count} request_items"
+            processed, unprocessed = get_batch_from_dynamo(batch)
+            results.concat(processed)
+
+            unless unprocessed.empty?
+              Dynamo::Record.logger.debug "#{name} batch_get_item #{unprocessed.count} unprocessed_keys"
+              unprocessed_keys.unshift(*unprocessed)
+              sleep(rand(1 << attempt) + 1)
+              attempt += 1
+            end
+          end
+
+          if !unprocessed_keys.empty? && attempt > max_retries
+            raise NumberOfRetriesExceeded.new(keys), 'Number of retries exceeded'
+          end
+
+          results.map { |r| build_item_from_hash(r) }
+        end
+
+        # BatchGetItem will return a partial result if provisioned throughput is
+        # exceeded or an internal error occurs. If all items cannot be processed
+        # ProvisionedThroughputExceededException is raised.
+        # https://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
+        def get_batch_from_dynamo(batch)
+          response = dynamodb_client.batch_get_item(
+            request_items: {
+              table_name => { keys: batch }
+            }
+          )
+          [
+            response.responses[table_name],
+            response.unprocessed_keys[table_name]&.keys || []
+          ]
+        rescue Aws::DynamoDB::Errors::ProvisionedThroughputExceededException
+          [[], batch]
+        end
+
+        # Convert response hashes to Record instances
+        # Adapted from `build_item_from_resp` used by `find` but that method is closely tied to get_item response format
+        # https://github.com/aws/aws-sdk-ruby-record/blob/v2.3.0/lib/aws-record/record/item_operations.rb#L593
+        def build_item_from_hash(resp)
+          record = new
+          data = record.instance_variable_get('@data')
+          attributes.attributes.each do |name, attr|
+            data.set_attribute(name, attr.extract(resp))
+            data.new_record = false
+          end
+          record
+        end
+      end
+
+      class NumberOfRetriesExceeded < RuntimeError
+        attr_reader :keys
+
+        def initialize(keys)
+          @keys = keys
+        end
+      end
+    end
+  end
+end

data/lib/dynamo/record/batch_write.rb

@@ -2,7 +2,6 @@ module Dynamo
   module Record
     module BatchWrite
       BATCH_SIZE = 25
-      DEFAULT_MAX_RETRIES = 5
       MAX_PAYLOAD_SIZE = 16_000_000 # 16 MB
 
       module ClassMethods
@@ -78,11 +77,6 @@ module Dynamo
             current_batch_size += request.request_size
           end
         end
-
-        # Override in model if you want it to be different.
-        def max_retries
-          DEFAULT_MAX_RETRIES
-        end
       end
 
       class NumberOfRetriesExceeded < RuntimeError

data/lib/dynamo/record/model.rb

@@ -2,6 +2,7 @@ module Dynamo
   module Record
     module Model
       COMPOSITE_DELIMITER = '|'.freeze
+      DEFAULT_MAX_RETRIES = 5
 
       def self.included(klass)
         klass.include(Aws::Record)
@@ -9,6 +10,7 @@ module Dynamo
 
         klass.extend ClassMethods
         klass.extend Dynamo::Record::BatchWrite::ClassMethods
+        klass.extend Dynamo::Record::BatchGet::ClassMethods
         klass.send :prepend, InstanceMethods
       end
 
@@ -104,7 +106,9 @@ module Dynamo
           string.split(COMPOSITE_DELIMITER)
         end
 
-        # TODO: Create a batch save method.
+        def max_retries
+          DEFAULT_MAX_RETRIES
+        end
       end
 
       module InstanceMethods

data/lib/dynamo/record/railtie.rb

@@ -3,6 +3,10 @@ module Dynamo
     class Railtie < Rails::Railtie
       railtie_name :dynamo_record
 
+      initializer 'logger' do
+        Dynamo::Record.logger = Rails.logger
+      end
+
       rake_tasks do
         load 'tasks/dynamo.rake'
       end

data/lib/dynamo/record/version.rb

@@ -1,5 +1,5 @@
 module Dynamo
   module Record
-    VERSION = '1.1.0'.freeze
+    VERSION = '1.2.0'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamo-record
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Davis McClellan
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-30 00:00:00.000000000 Z
+date: 2019-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -243,6 +243,7 @@ files:
 - docker-compose.yml
 - dynamo-record.gemspec
 - lib/dynamo/record.rb
+- lib/dynamo/record/batch_get.rb
 - lib/dynamo/record/batch_request.rb
 - lib/dynamo/record/batch_write.rb
 - lib/dynamo/record/marshalers.rb
@@ -278,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Extensions to Aws::Record for working with DynamoDB.