dynamo-record 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cc3a4024e4ef59d2f27311c320b5bf33aa91ffb4
4
- data.tar.gz: 793d770c42ac639e77a0ce1e4db3ca1fd6cef2fb
3
+ metadata.gz: e81e37de2e8ba6804662a756c1af90a60a85bc1e
4
+ data.tar.gz: 512d203880a9b118735f17fb66886578e5b3ed1a
5
5
  SHA512:
6
- metadata.gz: 19daa6e06995bfc7d57d4c05d5454e6def992562742edc9d20b0aea56fd8011f3ca8343f9b5dcba1c5ba21b8921aa1c3ec2ce644963273e0ab992e6dc1ead98b
7
- data.tar.gz: 9578863c9c36e8dfa7577af2f48b5f357eea6b7f9281c2964fa3bcde6134a02b87677769fcfde57c69ac990c808ab6f465f10304f9d84a55d5200ba68afe72b0
6
+ metadata.gz: c6dec5f3cef468bd4dcbb024c17d7e45a2cf55507c895fb66f082dc37bf9db55ab0ac6fa67f6a3f3215fcf88d708b03705235f660db8e5172b52f1b40eded2e8
7
+ data.tar.gz: b8eb760aef776b82c3ea5d1ded6cb02aca7201937cb390a9629e94002f73ce81974475386b15f85e05bf7b6ca5b571476bac93ab4d5b607d09843c26138b7ebb
@@ -1,15 +1,17 @@
1
+ /.gitignore
2
+ /.rspec_status
3
+ /.ruby-version
1
4
  /coverage/
2
5
  /doc/
3
6
  /Dockerfile
4
7
  /docker-compose.yml
5
8
  /docker-compose.override.example.yml
6
9
  /docker-compose.override.yml
10
+ /dynamo-record-*.gem
7
11
  /Gemfile.lock
8
12
  /log/
9
13
  /pkg/
10
14
  /spec/gemfiles/.bundle/
11
- /spec/dummy/log/
12
- /spec/dummy/tmp/
13
15
  /spec/gemfiles/*.gemfile.lock
14
16
  /switchman-inst-jobs-*.gem
15
17
  /tmp/
data/README.md CHANGED
@@ -241,11 +241,10 @@ cp docker-compose.override.example.yml docker-compose.override.yml
241
241
 
242
242
  ## Making a new Release
243
243
 
244
- To install this gem onto your local machine, run `bundle exec rake install`. To
245
- release a new version, update the version number in `version.rb`, and then just
246
- run `bundle exec rake release`, which will create a git tag for the version,
247
- push git commits and tags, and push the `.gem` file to
248
- [rubygems.org](https://rubygems.org).
244
+ To release a new version, update the version number in `version.rb`, and then
245
+ just run `gem build dynamo-record`, and push the `.gem` file to
246
+ [rubygems.org](https://rubygems.org). To install this gem onto your local
247
+ machine, run `gem install dynamo-record-*.gem`.
249
248
 
250
249
 
251
250
  ## Contributing
data/build.sh CHANGED
@@ -1,5 +1,7 @@
1
1
  #!/bin/bash -ex
2
2
 
3
+ COMPOSE_FILE="docker-compose.yml"
4
+
3
5
  function cleanup() {
4
6
  exit_code=$?
5
7
  set +e
@@ -2,9 +2,7 @@ version: '2'
2
2
 
3
3
  services:
4
4
  app:
5
- build:
6
- context: .
7
- dockerfile: Dockerfile
5
+ build: .
8
6
  environment:
9
7
  AWS_ACCESS_KEY_ID: x
10
8
  AWS_SECRET_ACCESS_KEY: x
@@ -3,6 +3,8 @@ require 'aws-record'
3
3
  require 'rails/railtie'
4
4
 
5
5
  require 'dynamo/record/marshalers'
6
+ require 'dynamo/record/batch_write'
7
+ require 'dynamo/record/batch_request'
6
8
  require 'dynamo/record/model'
7
9
  require 'dynamo/record/railtie'
8
10
  require 'dynamo/record/model_existence_validator'
@@ -0,0 +1,59 @@
1
+ module Dynamo
2
+ module Record
3
+ class BatchRequest
4
+ MAX_RECORD_SIZE = 400_000 # 400 KB
5
+
6
+ attr_reader :record, :type
7
+
8
+ def initialize(record, type)
9
+ @type = type
10
+ @record = record
11
+ validate_type
12
+ validate_request_size
13
+ end
14
+
15
+ def request
16
+ @_request ||= send("#{type}_request")
17
+ end
18
+
19
+ def save_request
20
+ {
21
+ put_request: {
22
+ item: record.send(:_build_item_for_save)
23
+ }
24
+ }
25
+ end
26
+
27
+ def delete_request
28
+ {
29
+ delete_request: {
30
+ key: record.send(:key_values)
31
+ }
32
+ }
33
+ end
34
+
35
+ def request_size
36
+ @_request_size ||= request.to_json.length
37
+ end
38
+
39
+ def validate_request_size
40
+ return unless request_size > MAX_RECORD_SIZE
41
+ raise RecordTooLargeError.new(record), 'Record is too large'
42
+ end
43
+
44
+ def validate_type
45
+ return if %i[save delete].include? type
46
+ raise UnsupportedRequestTypeError
47
+ end
48
+ end
49
+
50
+ class UnsupportedRequestTypeError < RuntimeError; end
51
+ class RecordTooLargeError < RuntimeError
52
+ attr_reader :record
53
+
54
+ def initialize(record)
55
+ @record = record
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,96 @@
1
+ module Dynamo
2
+ module Record
3
+ module BatchWrite
4
+ BATCH_SIZE = 25
5
+ DEFAULT_MAX_RETRIES = 5
6
+ MAX_PAYLOAD_SIZE = 16_000_000 # 16 MB
7
+
8
+ module ClassMethods
9
+ def batch_save!(records)
10
+ batch_method(records, :save)
11
+ end
12
+
13
+ def batch_delete!(records)
14
+ batch_method(records, :delete)
15
+ end
16
+
17
+ private
18
+
19
+ def batch_method(records, type)
20
+ requests = records.map do |record|
21
+ BatchRequest.new(record, type)
22
+ end
23
+ process_batch_write_requests(requests)
24
+ end
25
+
26
+ def process_batch_write_requests(requests, retry_count = 1)
27
+ # Need to batch with every request. This allows for unprocessed items
28
+ # to be batched in with the rest of the requests.
29
+ batched_requests = pre_batch(requests)
30
+
31
+ # Need to start the exponential backoff at the first sign of a failed request
32
+ request_failed = false
33
+
34
+ # TODO: Can further optimize this request with parallelization
35
+ # i.e. `unprocessed_items = Parallel.map(batched_requests) do |batch|`
36
+ # Note that `Parallel` doesn't seem to play well with Rspec mocks.
37
+ unprocessed_items = batched_requests.each_with_object([]) do |batch, unprocessed|
38
+ # Returns the batch as the defacto unprocessed items if one request
39
+ # has already failed. This will cut down on the number of
40
+ # `ProvisionedThroughputExceededException`s seen
41
+ if request_failed
42
+ unprocessed << batch
43
+ else
44
+ response = write_batch_to_dynamo(batch)
45
+ if response.present?
46
+ request_failed = true
47
+ unprocessed << response
48
+ end
49
+ end
50
+ end
51
+ reprocess_items(unprocessed_items.flatten, retry_count) if unprocessed_items.present?
52
+ end
53
+
54
+ def write_batch_to_dynamo(batch)
55
+ response = dynamodb_client.batch_write_item(request_items: { table_name => batch.map(&:request) })
56
+ batch.select { |i| response.unprocessed_items[table_name]&.map(&:to_h)&.include?(i.request) }
57
+ rescue Aws::DynamoDB::Errors::ProvisionedThroughputExceededException
58
+ batch
59
+ end
60
+
61
+ def reprocess_items(unprocessed_items, retry_count)
62
+ if retry_count > max_retries
63
+ raise NumberOfRetriesExceeded.new(unprocessed_items.map(&:record)), 'Number of retries exceeded'
64
+ end
65
+ sleep(rand(1 << retry_count) + 1)
66
+ process_batch_write_requests(unprocessed_items, retry_count + 1)
67
+ end
68
+
69
+ def pre_batch(requests)
70
+ current_batch_size = 0
71
+ requests.each_with_object([[]]) do |request, batches|
72
+ if batches.last.length >= BATCH_SIZE || current_batch_size + request.request_size > MAX_PAYLOAD_SIZE
73
+ batches.push([])
74
+ current_batch_size = 0
75
+ end
76
+ batches.last.push(request)
77
+ current_batch_size += request.request_size
78
+ end
79
+ end
80
+
81
+ # Override in model if you want it to be different.
82
+ def max_retries
83
+ DEFAULT_MAX_RETRIES
84
+ end
85
+ end
86
+
87
+ class NumberOfRetriesExceeded < RuntimeError
88
+ attr_reader :unprocessed_items
89
+
90
+ def initialize(unprocessed_items)
91
+ @unprocessed_items = unprocessed_items
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -8,6 +8,7 @@ module Dynamo
8
8
  klass.include(Dynamo::Record::Marshalers)
9
9
 
10
10
  klass.extend ClassMethods
11
+ klass.extend Dynamo::Record::BatchWrite::ClassMethods
11
12
  klass.send :prepend, InstanceMethods
12
13
  end
13
14
 
@@ -1,5 +1,5 @@
1
1
  module Dynamo
2
2
  module Record
3
- VERSION = '0.3.0'.freeze
3
+ VERSION = '0.4.0'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dynamo-record
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Davis McClellan
@@ -14,7 +14,7 @@ authors:
14
14
  autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
- date: 2017-09-22 00:00:00.000000000 Z
17
+ date: 2018-01-09 00:00:00.000000000 Z
18
18
  dependencies:
19
19
  - !ruby/object:Gem::Dependency
20
20
  name: aws-record
@@ -243,6 +243,8 @@ files:
243
243
  - docker-compose.yml
244
244
  - dynamo-record.gemspec
245
245
  - lib/dynamo/record.rb
246
+ - lib/dynamo/record/batch_request.rb
247
+ - lib/dynamo/record/batch_write.rb
246
248
  - lib/dynamo/record/marshalers.rb
247
249
  - lib/dynamo/record/model.rb
248
250
  - lib/dynamo/record/model_existence_validator.rb
@@ -276,8 +278,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
276
278
  version: '0'
277
279
  requirements: []
278
280
  rubyforge_project:
279
- rubygems_version: 2.6.11
281
+ rubygems_version: 2.6.13
280
282
  signing_key:
281
283
  specification_version: 4
282
284
  summary: Extensions to Aws::Record for working with DynamoDB.
283
285
  test_files: []
286
+ has_rdoc: