dynamo-record 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/dynamo/record.rb +13 -0
- data/lib/dynamo/record/batch_get.rb +81 -0
- data/lib/dynamo/record/batch_write.rb +0 -6
- data/lib/dynamo/record/model.rb +5 -1
- data/lib/dynamo/record/railtie.rb +4 -0
- data/lib/dynamo/record/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e0e65c5e7c2ecaac423640b5872a8c40d58a539a
|
4
|
+
data.tar.gz: 807766fdecdccdd5029a11a1f0fcbad8a2e61569
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5213186128c25aca4a3a37a27ee8534f7054f9bac3e9908c930a857d43811af1628a66f5ff76a551190a3144af6d98c464678c8becf6859d8cd8fa61e78d67d7
|
7
|
+
data.tar.gz: efefa9924786b3068896273ebbe3711d1eb2a7872e5270b15e735e81fc8cf9051b304f1fd3d036c17a7707cf252bdf6717bb48416c81f0c03025204c46aefc48
|
data/lib/dynamo/record.rb
CHANGED
@@ -3,6 +3,7 @@ require 'aws-record'
|
|
3
3
|
require 'rails/railtie'
|
4
4
|
|
5
5
|
require 'dynamo/record/marshalers'
|
6
|
+
require 'dynamo/record/batch_get'
|
6
7
|
require 'dynamo/record/batch_write'
|
7
8
|
require 'dynamo/record/batch_request'
|
8
9
|
require 'dynamo/record/model'
|
@@ -17,3 +18,15 @@ require 'dynamo/record/task_helpers/drop_table'
|
|
17
18
|
require 'dynamo/record/task_helpers/list_tables'
|
18
19
|
require 'dynamo/record/task_helpers/migration_runner'
|
19
20
|
require 'dynamo/record/task_helpers/scale'
|
21
|
+
|
22
|
+
module Dynamo
|
23
|
+
module Record
|
24
|
+
class << self
|
25
|
+
attr_writer :logger
|
26
|
+
|
27
|
+
def logger
|
28
|
+
@logger ||= Logger.new($stdout).tap { |log| log.progname = self.name }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Dynamo
|
4
|
+
module Record
|
5
|
+
module BatchGet
|
6
|
+
BATCH_SIZE = 100
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def batch_get(keys)
|
10
|
+
process_batch_get_requests(keys)
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def process_batch_get_requests(keys)
|
16
|
+
attempt = 1
|
17
|
+
results = []
|
18
|
+
unprocessed_keys = keys.dup.map(&:stringify_keys)
|
19
|
+
while !unprocessed_keys.empty? && attempt <= max_retries
|
20
|
+
batch = unprocessed_keys.shift(BATCH_SIZE)
|
21
|
+
Dynamo::Record.logger.debug "#{name} batch_get_item #{batch.count} request_items"
|
22
|
+
processed, unprocessed = get_batch_from_dynamo(batch)
|
23
|
+
results.concat(processed)
|
24
|
+
|
25
|
+
unless unprocessed.empty?
|
26
|
+
Dynamo::Record.logger.debug "#{name} batch_get_item #{unprocessed.count} unprocessed_keys"
|
27
|
+
unprocessed_keys.unshift(*unprocessed)
|
28
|
+
sleep(rand(1 << attempt) + 1)
|
29
|
+
attempt += 1
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
if !unprocessed_keys.empty? && attempt > max_retries
|
34
|
+
raise NumberOfRetriesExceeded.new(keys), 'Number of retries exceeded'
|
35
|
+
end
|
36
|
+
|
37
|
+
results.map { |r| build_item_from_hash(r) }
|
38
|
+
end
|
39
|
+
|
40
|
+
# BatchGetItem will return a partial result if provisioned throughput is
|
41
|
+
# exceeded or an internal error occurs. If all items cannot be processed
|
42
|
+
# ProvisionedThroughputExceededException is raised.
|
43
|
+
# https://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
|
44
|
+
def get_batch_from_dynamo(batch)
|
45
|
+
response = dynamodb_client.batch_get_item(
|
46
|
+
request_items: {
|
47
|
+
table_name => { keys: batch }
|
48
|
+
}
|
49
|
+
)
|
50
|
+
[
|
51
|
+
response.responses[table_name],
|
52
|
+
response.unprocessed_keys[table_name]&.keys || []
|
53
|
+
]
|
54
|
+
rescue Aws::DynamoDB::Errors::ProvisionedThroughputExceededException
|
55
|
+
[[], batch]
|
56
|
+
end
|
57
|
+
|
58
|
+
# Convert response hashes to Record instances
|
59
|
+
# Adapted from `build_item_from_resp` used by `find` but that method is closely tied to get_item response format
|
60
|
+
# https://github.com/aws/aws-sdk-ruby-record/blob/v2.3.0/lib/aws-record/record/item_operations.rb#L593
|
61
|
+
def build_item_from_hash(resp)
|
62
|
+
record = new
|
63
|
+
data = record.instance_variable_get('@data')
|
64
|
+
attributes.attributes.each do |name, attr|
|
65
|
+
data.set_attribute(name, attr.extract(resp))
|
66
|
+
data.new_record = false
|
67
|
+
end
|
68
|
+
record
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
class NumberOfRetriesExceeded < RuntimeError
|
73
|
+
attr_reader :keys
|
74
|
+
|
75
|
+
def initialize(keys)
|
76
|
+
@keys = keys
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -2,7 +2,6 @@ module Dynamo
|
|
2
2
|
module Record
|
3
3
|
module BatchWrite
|
4
4
|
BATCH_SIZE = 25
|
5
|
-
DEFAULT_MAX_RETRIES = 5
|
6
5
|
MAX_PAYLOAD_SIZE = 16_000_000 # 16 MB
|
7
6
|
|
8
7
|
module ClassMethods
|
@@ -78,11 +77,6 @@ module Dynamo
|
|
78
77
|
current_batch_size += request.request_size
|
79
78
|
end
|
80
79
|
end
|
81
|
-
|
82
|
-
# Override in model if you want it to be different.
|
83
|
-
def max_retries
|
84
|
-
DEFAULT_MAX_RETRIES
|
85
|
-
end
|
86
80
|
end
|
87
81
|
|
88
82
|
class NumberOfRetriesExceeded < RuntimeError
|
data/lib/dynamo/record/model.rb
CHANGED
@@ -2,6 +2,7 @@ module Dynamo
|
|
2
2
|
module Record
|
3
3
|
module Model
|
4
4
|
COMPOSITE_DELIMITER = '|'.freeze
|
5
|
+
DEFAULT_MAX_RETRIES = 5
|
5
6
|
|
6
7
|
def self.included(klass)
|
7
8
|
klass.include(Aws::Record)
|
@@ -9,6 +10,7 @@ module Dynamo
|
|
9
10
|
|
10
11
|
klass.extend ClassMethods
|
11
12
|
klass.extend Dynamo::Record::BatchWrite::ClassMethods
|
13
|
+
klass.extend Dynamo::Record::BatchGet::ClassMethods
|
12
14
|
klass.send :prepend, InstanceMethods
|
13
15
|
end
|
14
16
|
|
@@ -104,7 +106,9 @@ module Dynamo
|
|
104
106
|
string.split(COMPOSITE_DELIMITER)
|
105
107
|
end
|
106
108
|
|
107
|
-
|
109
|
+
def max_retries
|
110
|
+
DEFAULT_MAX_RETRIES
|
111
|
+
end
|
108
112
|
end
|
109
113
|
|
110
114
|
module InstanceMethods
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dynamo-record
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Davis McClellan
|
@@ -14,7 +14,7 @@ authors:
|
|
14
14
|
autorequire:
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date:
|
17
|
+
date: 2019-03-01 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: activemodel
|
@@ -243,6 +243,7 @@ files:
|
|
243
243
|
- docker-compose.yml
|
244
244
|
- dynamo-record.gemspec
|
245
245
|
- lib/dynamo/record.rb
|
246
|
+
- lib/dynamo/record/batch_get.rb
|
246
247
|
- lib/dynamo/record/batch_request.rb
|
247
248
|
- lib/dynamo/record/batch_write.rb
|
248
249
|
- lib/dynamo/record/marshalers.rb
|
@@ -278,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
278
279
|
version: '0'
|
279
280
|
requirements: []
|
280
281
|
rubyforge_project:
|
281
|
-
rubygems_version: 2.
|
282
|
+
rubygems_version: 2.5.1
|
282
283
|
signing_key:
|
283
284
|
specification_version: 4
|
284
285
|
summary: Extensions to Aws::Record for working with DynamoDB.
|