dynamo-record 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dockerignore +4 -2
- data/README.md +4 -5
- data/build.sh +2 -0
- data/docker-compose.yml +1 -3
- data/lib/dynamo/record.rb +2 -0
- data/lib/dynamo/record/batch_request.rb +59 -0
- data/lib/dynamo/record/batch_write.rb +96 -0
- data/lib/dynamo/record/model.rb +1 -0
- data/lib/dynamo/record/version.rb +1 -1
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e81e37de2e8ba6804662a756c1af90a60a85bc1e
|
4
|
+
data.tar.gz: 512d203880a9b118735f17fb66886578e5b3ed1a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c6dec5f3cef468bd4dcbb024c17d7e45a2cf55507c895fb66f082dc37bf9db55ab0ac6fa67f6a3f3215fcf88d708b03705235f660db8e5172b52f1b40eded2e8
|
7
|
+
data.tar.gz: b8eb760aef776b82c3ea5d1ded6cb02aca7201937cb390a9629e94002f73ce81974475386b15f85e05bf7b6ca5b571476bac93ab4d5b607d09843c26138b7ebb
|
data/.dockerignore
CHANGED
@@ -1,15 +1,17 @@
|
|
1
|
+
/.gitignore
|
2
|
+
/.rspec_status
|
3
|
+
/.ruby-version
|
1
4
|
/coverage/
|
2
5
|
/doc/
|
3
6
|
/Dockerfile
|
4
7
|
/docker-compose.yml
|
5
8
|
/docker-compose.override.example.yml
|
6
9
|
/docker-compose.override.yml
|
10
|
+
/dynamo-record-*.gem
|
7
11
|
/Gemfile.lock
|
8
12
|
/log/
|
9
13
|
/pkg/
|
10
14
|
/spec/gemfiles/.bundle/
|
11
|
-
/spec/dummy/log/
|
12
|
-
/spec/dummy/tmp/
|
13
15
|
/spec/gemfiles/*.gemfile.lock
|
14
16
|
/switchman-inst-jobs-*.gem
|
15
17
|
/tmp/
|
data/README.md
CHANGED
@@ -241,11 +241,10 @@ cp docker-compose.override.example.yml docker-compose.override.yml
|
|
241
241
|
|
242
242
|
## Making a new Release
|
243
243
|
|
244
|
-
To
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
[rubygems.org](https://rubygems.org).
|
244
|
+
To release a new version, update the version number in `version.rb`, and then
|
245
|
+
just run `gem build dynamo-record`, and push the `.gem` file to
|
246
|
+
[rubygems.org](https://rubygems.org). To install this gem onto your local
|
247
|
+
machine, run `gem install dynamo-record-*.gem`.
|
249
248
|
|
250
249
|
|
251
250
|
## Contributing
|
data/build.sh
CHANGED
data/docker-compose.yml
CHANGED
data/lib/dynamo/record.rb
CHANGED
@@ -3,6 +3,8 @@ require 'aws-record'
|
|
3
3
|
require 'rails/railtie'
|
4
4
|
|
5
5
|
require 'dynamo/record/marshalers'
|
6
|
+
require 'dynamo/record/batch_write'
|
7
|
+
require 'dynamo/record/batch_request'
|
6
8
|
require 'dynamo/record/model'
|
7
9
|
require 'dynamo/record/railtie'
|
8
10
|
require 'dynamo/record/model_existence_validator'
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module Dynamo
|
2
|
+
module Record
|
3
|
+
class BatchRequest
|
4
|
+
MAX_RECORD_SIZE = 400_000 # 400 KB
|
5
|
+
|
6
|
+
attr_reader :record, :type
|
7
|
+
|
8
|
+
def initialize(record, type)
|
9
|
+
@type = type
|
10
|
+
@record = record
|
11
|
+
validate_type
|
12
|
+
validate_request_size
|
13
|
+
end
|
14
|
+
|
15
|
+
def request
|
16
|
+
@_request ||= send("#{type}_request")
|
17
|
+
end
|
18
|
+
|
19
|
+
def save_request
|
20
|
+
{
|
21
|
+
put_request: {
|
22
|
+
item: record.send(:_build_item_for_save)
|
23
|
+
}
|
24
|
+
}
|
25
|
+
end
|
26
|
+
|
27
|
+
def delete_request
|
28
|
+
{
|
29
|
+
delete_request: {
|
30
|
+
key: record.send(:key_values)
|
31
|
+
}
|
32
|
+
}
|
33
|
+
end
|
34
|
+
|
35
|
+
def request_size
|
36
|
+
@_request_size ||= request.to_json.length
|
37
|
+
end
|
38
|
+
|
39
|
+
def validate_request_size
|
40
|
+
return unless request_size > MAX_RECORD_SIZE
|
41
|
+
raise RecordTooLargeError.new(record), 'Record is too large'
|
42
|
+
end
|
43
|
+
|
44
|
+
def validate_type
|
45
|
+
return if %i[save delete].include? type
|
46
|
+
raise UnsupportedRequestTypeError
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
class UnsupportedRequestTypeError < RuntimeError; end
|
51
|
+
class RecordTooLargeError < RuntimeError
|
52
|
+
attr_reader :record
|
53
|
+
|
54
|
+
def initialize(record)
|
55
|
+
@record = record
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
module Dynamo
|
2
|
+
module Record
|
3
|
+
module BatchWrite
|
4
|
+
BATCH_SIZE = 25
|
5
|
+
DEFAULT_MAX_RETRIES = 5
|
6
|
+
MAX_PAYLOAD_SIZE = 16_000_000 # 16 MB
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def batch_save!(records)
|
10
|
+
batch_method(records, :save)
|
11
|
+
end
|
12
|
+
|
13
|
+
def batch_delete!(records)
|
14
|
+
batch_method(records, :delete)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def batch_method(records, type)
|
20
|
+
requests = records.map do |record|
|
21
|
+
BatchRequest.new(record, type)
|
22
|
+
end
|
23
|
+
process_batch_write_requests(requests)
|
24
|
+
end
|
25
|
+
|
26
|
+
def process_batch_write_requests(requests, retry_count = 1)
|
27
|
+
# Need to batch with every request. This allows for unprocessed items
|
28
|
+
# to be batched in with the rest of the requests.
|
29
|
+
batched_requests = pre_batch(requests)
|
30
|
+
|
31
|
+
# Need to start the exponential backoff at the first sign of a failed request
|
32
|
+
request_failed = false
|
33
|
+
|
34
|
+
# TODO: Can further optimize this request with parallelization
|
35
|
+
# i.e. `unprocessed_items = Parallel.map(batched_requests) do |batch|`
|
36
|
+
# Note that `Parallel` doesn't seem to play well with Rspec mocks.
|
37
|
+
unprocessed_items = batched_requests.each_with_object([]) do |batch, unprocessed|
|
38
|
+
# Returns the batch as the defacto unprocessed items if one request
|
39
|
+
# has already failed. This will cut down on the number of
|
40
|
+
# `ProvisionedThroughputExceededException`s seen
|
41
|
+
if request_failed
|
42
|
+
unprocessed << batch
|
43
|
+
else
|
44
|
+
response = write_batch_to_dynamo(batch)
|
45
|
+
if response.present?
|
46
|
+
request_failed = true
|
47
|
+
unprocessed << response
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
reprocess_items(unprocessed_items.flatten, retry_count) if unprocessed_items.present?
|
52
|
+
end
|
53
|
+
|
54
|
+
def write_batch_to_dynamo(batch)
|
55
|
+
response = dynamodb_client.batch_write_item(request_items: { table_name => batch.map(&:request) })
|
56
|
+
batch.select { |i| response.unprocessed_items[table_name]&.map(&:to_h)&.include?(i.request) }
|
57
|
+
rescue Aws::DynamoDB::Errors::ProvisionedThroughputExceededException
|
58
|
+
batch
|
59
|
+
end
|
60
|
+
|
61
|
+
def reprocess_items(unprocessed_items, retry_count)
|
62
|
+
if retry_count > max_retries
|
63
|
+
raise NumberOfRetriesExceeded.new(unprocessed_items.map(&:record)), 'Number of retries exceeded'
|
64
|
+
end
|
65
|
+
sleep(rand(1 << retry_count) + 1)
|
66
|
+
process_batch_write_requests(unprocessed_items, retry_count + 1)
|
67
|
+
end
|
68
|
+
|
69
|
+
def pre_batch(requests)
|
70
|
+
current_batch_size = 0
|
71
|
+
requests.each_with_object([[]]) do |request, batches|
|
72
|
+
if batches.last.length >= BATCH_SIZE || current_batch_size + request.request_size > MAX_PAYLOAD_SIZE
|
73
|
+
batches.push([])
|
74
|
+
current_batch_size = 0
|
75
|
+
end
|
76
|
+
batches.last.push(request)
|
77
|
+
current_batch_size += request.request_size
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# Override in model if you want it to be different.
|
82
|
+
def max_retries
|
83
|
+
DEFAULT_MAX_RETRIES
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
class NumberOfRetriesExceeded < RuntimeError
|
88
|
+
attr_reader :unprocessed_items
|
89
|
+
|
90
|
+
def initialize(unprocessed_items)
|
91
|
+
@unprocessed_items = unprocessed_items
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
data/lib/dynamo/record/model.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dynamo-record
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Davis McClellan
|
@@ -14,7 +14,7 @@ authors:
|
|
14
14
|
autorequire:
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date:
|
17
|
+
date: 2018-01-09 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: aws-record
|
@@ -243,6 +243,8 @@ files:
|
|
243
243
|
- docker-compose.yml
|
244
244
|
- dynamo-record.gemspec
|
245
245
|
- lib/dynamo/record.rb
|
246
|
+
- lib/dynamo/record/batch_request.rb
|
247
|
+
- lib/dynamo/record/batch_write.rb
|
246
248
|
- lib/dynamo/record/marshalers.rb
|
247
249
|
- lib/dynamo/record/model.rb
|
248
250
|
- lib/dynamo/record/model_existence_validator.rb
|
@@ -276,8 +278,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
276
278
|
version: '0'
|
277
279
|
requirements: []
|
278
280
|
rubyforge_project:
|
279
|
-
rubygems_version: 2.6.
|
281
|
+
rubygems_version: 2.6.13
|
280
282
|
signing_key:
|
281
283
|
specification_version: 4
|
282
284
|
summary: Extensions to Aws::Record for working with DynamoDB.
|
283
285
|
test_files: []
|
286
|
+
has_rdoc:
|