dynamo-record 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc3a4024e4ef59d2f27311c320b5bf33aa91ffb4
-  data.tar.gz: 793d770c42ac639e77a0ce1e4db3ca1fd6cef2fb
+  metadata.gz: e81e37de2e8ba6804662a756c1af90a60a85bc1e
+  data.tar.gz: 512d203880a9b118735f17fb66886578e5b3ed1a
 SHA512:
-  metadata.gz: 19daa6e06995bfc7d57d4c05d5454e6def992562742edc9d20b0aea56fd8011f3ca8343f9b5dcba1c5ba21b8921aa1c3ec2ce644963273e0ab992e6dc1ead98b
-  data.tar.gz: 9578863c9c36e8dfa7577af2f48b5f357eea6b7f9281c2964fa3bcde6134a02b87677769fcfde57c69ac990c808ab6f465f10304f9d84a55d5200ba68afe72b0
+  metadata.gz: c6dec5f3cef468bd4dcbb024c17d7e45a2cf55507c895fb66f082dc37bf9db55ab0ac6fa67f6a3f3215fcf88d708b03705235f660db8e5172b52f1b40eded2e8
+  data.tar.gz: b8eb760aef776b82c3ea5d1ded6cb02aca7201937cb390a9629e94002f73ce81974475386b15f85e05bf7b6ca5b571476bac93ab4d5b607d09843c26138b7ebb

data/.dockerignore

@@ -1,15 +1,17 @@
+/.gitignore
+/.rspec_status
+/.ruby-version
 /coverage/
 /doc/
 /Dockerfile
 /docker-compose.yml
 /docker-compose.override.example.yml
 /docker-compose.override.yml
+/dynamo-record-*.gem
 /Gemfile.lock
 /log/
 /pkg/
 /spec/gemfiles/.bundle/
-/spec/dummy/log/
-/spec/dummy/tmp/
 /spec/gemfiles/*.gemfile.lock
 /switchman-inst-jobs-*.gem
 /tmp/

data/README.md

@@ -241,11 +241,10 @@ cp docker-compose.override.example.yml docker-compose.override.yml
 
 ## Making a new Release
 
-To install this gem onto your local machine, run `bundle exec rake install`. To
-release a new version, update the version number in `version.rb`, and then just
-run `bundle exec rake release`, which will create a git tag for the version,
-push git commits and tags, and push the `.gem` file to
-[rubygems.org](https://rubygems.org).
+To release a new version, update the version number in `version.rb`, and then
+just run `gem build dynamo-record`, and push the `.gem` file to
+[rubygems.org](https://rubygems.org). To install this gem onto your local
+machine, run `gem install dynamo-record-*.gem`.
 
 
 ## Contributing

data/build.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -ex
 
+COMPOSE_FILE="docker-compose.yml"
+
 function cleanup() {
   exit_code=$?
   set +e

data/docker-compose.yml

@@ -2,9 +2,7 @@ version: '2'
 
 services:
   app:
-    build:
-      context: .
-      dockerfile: Dockerfile
+    build: .
     environment:
       AWS_ACCESS_KEY_ID: x
       AWS_SECRET_ACCESS_KEY: x

data/lib/dynamo/record.rb

@@ -3,6 +3,8 @@ require 'aws-record'
 require 'rails/railtie'
 
 require 'dynamo/record/marshalers'
+require 'dynamo/record/batch_write'
+require 'dynamo/record/batch_request'
 require 'dynamo/record/model'
 require 'dynamo/record/railtie'
 require 'dynamo/record/model_existence_validator'

data/lib/dynamo/record/batch_request.rb

@@ -0,0 +1,59 @@
+module Dynamo
+  module Record
+    class BatchRequest
+      MAX_RECORD_SIZE = 400_000 # 400 KB
+
+      attr_reader :record, :type
+
+      def initialize(record, type)
+        @type = type
+        @record = record
+        validate_type
+        validate_request_size
+      end
+
+      def request
+        @_request ||= send("#{type}_request")
+      end
+
+      def save_request
+        {
+          put_request: {
+            item: record.send(:_build_item_for_save)
+          }
+        }
+      end
+
+      def delete_request
+        {
+          delete_request: {
+            key: record.send(:key_values)
+          }
+        }
+      end
+
+      def request_size
+        @_request_size ||= request.to_json.length
+      end
+
+      def validate_request_size
+        return unless request_size > MAX_RECORD_SIZE
+        raise RecordTooLargeError.new(record), 'Record is too large'
+      end
+
+      def validate_type
+        return if %i[save delete].include? type
+        raise UnsupportedRequestTypeError
+      end
+    end
+
+    class UnsupportedRequestTypeError < RuntimeError; end
+    class RecordTooLargeError < RuntimeError
+      attr_reader :record
+
+      def initialize(record)
+        @record = record
+      end
+    end
+  end
+end

data/lib/dynamo/record/batch_write.rb

@@ -0,0 +1,96 @@
+module Dynamo
+  module Record
+    module BatchWrite
+      BATCH_SIZE = 25
+      DEFAULT_MAX_RETRIES = 5
+      MAX_PAYLOAD_SIZE = 16_000_000 # 16 MB
+
+      module ClassMethods
+        def batch_save!(records)
+          batch_method(records, :save)
+        end
+
+        def batch_delete!(records)
+          batch_method(records, :delete)
+        end
+
+        private
+
+        def batch_method(records, type)
+          requests = records.map do |record|
+            BatchRequest.new(record, type)
+          end
+          process_batch_write_requests(requests)
+        end
+
+        def process_batch_write_requests(requests, retry_count = 1)
+          # Need to batch with every request. This allows for unprocessed items
+          # to be batched in with the rest of the requests.
+          batched_requests = pre_batch(requests)
+
+          # Need to start the exponential backoff at the first sign of a failed request
+          request_failed = false
+
+          # TODO: Can further optimize this request with parallelization
+          # i.e. `unprocessed_items = Parallel.map(batched_requests) do |batch|`
+          # Note that `Parallel` doesn't seem to play well with Rspec mocks.
+          unprocessed_items = batched_requests.each_with_object([]) do |batch, unprocessed|
+            # Returns the batch as the defacto unprocessed items if one request
+            # has already failed. This will cut down on the number of
+            # `ProvisionedThroughputExceededException`s seen
+            if request_failed
+              unprocessed << batch
+            else
+              response = write_batch_to_dynamo(batch)
+              if response.present?
+                request_failed = true
+                unprocessed << response
+              end
+            end
+          end
+          reprocess_items(unprocessed_items.flatten, retry_count) if unprocessed_items.present?
+        end
+
+        def write_batch_to_dynamo(batch)
+          response = dynamodb_client.batch_write_item(request_items: { table_name => batch.map(&:request) })
+          batch.select { |i| response.unprocessed_items[table_name]&.map(&:to_h)&.include?(i.request) }
+        rescue Aws::DynamoDB::Errors::ProvisionedThroughputExceededException
+          batch
+        end
+
+        def reprocess_items(unprocessed_items, retry_count)
+          if retry_count > max_retries
+            raise NumberOfRetriesExceeded.new(unprocessed_items.map(&:record)), 'Number of retries exceeded'
+          end
+          sleep(rand(1 << retry_count) + 1)
+          process_batch_write_requests(unprocessed_items, retry_count + 1)
+        end
+
+        def pre_batch(requests)
+          current_batch_size = 0
+          requests.each_with_object([[]]) do |request, batches|
+            if batches.last.length >= BATCH_SIZE || current_batch_size + request.request_size > MAX_PAYLOAD_SIZE
+              batches.push([])
+              current_batch_size = 0
+            end
+            batches.last.push(request)
+            current_batch_size += request.request_size
+          end
+        end
+
+        # Override in model if you want it to be different.
+        def max_retries
+          DEFAULT_MAX_RETRIES
+        end
+      end
+
+      class NumberOfRetriesExceeded < RuntimeError
+        attr_reader :unprocessed_items
+
+        def initialize(unprocessed_items)
+          @unprocessed_items = unprocessed_items
+        end
+      end
+    end
+  end
+end

data/lib/dynamo/record/model.rb

@@ -8,6 +8,7 @@ module Dynamo
         klass.include(Dynamo::Record::Marshalers)
 
         klass.extend ClassMethods
+        klass.extend Dynamo::Record::BatchWrite::ClassMethods
         klass.send :prepend, InstanceMethods
       end
 

data/lib/dynamo/record/version.rb

@@ -1,5 +1,5 @@
 module Dynamo
   module Record
-    VERSION = '0.3.0'.freeze
+    VERSION = '0.4.0'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamo-record
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Davis McClellan
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-22 00:00:00.000000000 Z
+date: 2018-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-record
@@ -243,6 +243,8 @@ files:
 - docker-compose.yml
 - dynamo-record.gemspec
 - lib/dynamo/record.rb
+- lib/dynamo/record/batch_request.rb
+- lib/dynamo/record/batch_write.rb
 - lib/dynamo/record/marshalers.rb
 - lib/dynamo/record/model.rb
 - lib/dynamo/record/model_existence_validator.rb
@@ -276,8 +278,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Extensions to Aws::Record for working with DynamoDB.
 test_files: []
+has_rdoc: