dynamoid 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +9 -6
- data/Appraisals +8 -14
- data/CHANGELOG.md +24 -0
- data/README.md +493 -228
- data/gemfiles/rails_4_2.gemfile +5 -7
- data/gemfiles/rails_5_0.gemfile +4 -6
- data/gemfiles/rails_5_1.gemfile +4 -6
- data/gemfiles/rails_5_2.gemfile +4 -6
- data/gemfiles/rails_6_0.gemfile +8 -0
- data/lib/dynamoid.rb +1 -0
- data/lib/dynamoid/adapter.rb +3 -10
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb +25 -69
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/batch_get_item.rb +105 -0
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/create_table.rb +9 -4
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb +11 -4
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb +11 -3
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/until_past_table_status.rb +3 -2
- data/lib/dynamoid/components.rb +6 -3
- data/lib/dynamoid/config.rb +1 -0
- data/lib/dynamoid/criteria.rb +1 -1
- data/lib/dynamoid/criteria/chain.rb +33 -6
- data/lib/dynamoid/criteria/key_fields_detector.rb +101 -32
- data/lib/dynamoid/dirty.rb +186 -34
- data/lib/dynamoid/document.rb +8 -216
- data/lib/dynamoid/fields.rb +8 -0
- data/lib/dynamoid/loadable.rb +31 -0
- data/lib/dynamoid/persistence.rb +177 -85
- data/lib/dynamoid/persistence/import.rb +72 -0
- data/lib/dynamoid/persistence/save.rb +63 -0
- data/lib/dynamoid/persistence/update_fields.rb +62 -0
- data/lib/dynamoid/persistence/upsert.rb +60 -0
- data/lib/dynamoid/version.rb +1 -1
- metadata +9 -2
data/gemfiles/rails_4_2.gemfile
CHANGED
@@ -1,11 +1,9 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
# This file was generated by Appraisal
|
4
2
|
|
5
|
-
source
|
3
|
+
source "https://rubygems.org"
|
6
4
|
|
7
|
-
gem
|
8
|
-
gem
|
9
|
-
gem
|
5
|
+
gem "pry-byebug", platforms: :ruby
|
6
|
+
gem "activemodel", "~> 4.2.0"
|
7
|
+
gem "nokogiri", "~> 1.6.8"
|
10
8
|
|
11
|
-
gemspec path:
|
9
|
+
gemspec path: "../"
|
data/gemfiles/rails_5_0.gemfile
CHANGED
@@ -1,10 +1,8 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
# This file was generated by Appraisal
|
4
2
|
|
5
|
-
source
|
3
|
+
source "https://rubygems.org"
|
6
4
|
|
7
|
-
gem
|
8
|
-
gem
|
5
|
+
gem "pry-byebug", platforms: :ruby
|
6
|
+
gem "activemodel", "~> 5.0.0"
|
9
7
|
|
10
|
-
gemspec path:
|
8
|
+
gemspec path: "../"
|
data/gemfiles/rails_5_1.gemfile
CHANGED
@@ -1,10 +1,8 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
# This file was generated by Appraisal
|
4
2
|
|
5
|
-
source
|
3
|
+
source "https://rubygems.org"
|
6
4
|
|
7
|
-
gem
|
8
|
-
gem
|
5
|
+
gem "pry-byebug", platforms: :ruby
|
6
|
+
gem "activemodel", "~> 5.1.0"
|
9
7
|
|
10
|
-
gemspec path:
|
8
|
+
gemspec path: "../"
|
data/gemfiles/rails_5_2.gemfile
CHANGED
@@ -1,10 +1,8 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
# This file was generated by Appraisal
|
4
2
|
|
5
|
-
source
|
3
|
+
source "https://rubygems.org"
|
6
4
|
|
7
|
-
gem
|
8
|
-
gem
|
5
|
+
gem "pry-byebug", platforms: :ruby
|
6
|
+
gem "activemodel", "~> 5.2.0"
|
9
7
|
|
10
|
-
gemspec path:
|
8
|
+
gemspec path: "../"
|
data/lib/dynamoid.rb
CHANGED
data/lib/dynamoid/adapter.rb
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
# require only 'concurrent/atom' once this issue is resolved:
|
4
4
|
# https://github.com/ruby-concurrency/concurrent-ruby/pull/377
|
5
5
|
require 'concurrent'
|
6
|
+
require "dynamoid/adapter_plugin/aws_sdk_v3"
|
6
7
|
|
7
8
|
# encoding: utf-8
|
8
9
|
module Dynamoid
|
@@ -29,7 +30,7 @@ module Dynamoid
|
|
29
30
|
def adapter
|
30
31
|
unless @adapter_.value
|
31
32
|
adapter = self.class.adapter_plugin_class.new
|
32
|
-
adapter.connect!
|
33
|
+
adapter.connect!
|
33
34
|
@adapter_.compare_and_set(nil, adapter)
|
34
35
|
clear_cache!
|
35
36
|
end
|
@@ -142,11 +143,7 @@ module Dynamoid
|
|
142
143
|
#
|
143
144
|
# @since 0.2.0
|
144
145
|
define_method(m) do |*args, &blk|
|
145
|
-
|
146
|
-
benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
|
147
|
-
else
|
148
|
-
benchmark(m.to_s, *args) { adapter.send(m, *args) }
|
149
|
-
end
|
146
|
+
benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
|
150
147
|
end
|
151
148
|
end
|
152
149
|
|
@@ -179,10 +176,6 @@ module Dynamoid
|
|
179
176
|
end
|
180
177
|
|
181
178
|
def self.adapter_plugin_class
|
182
|
-
unless Dynamoid.const_defined?(:AdapterPlugin) && Dynamoid::AdapterPlugin.const_defined?(Dynamoid::Config.adapter.camelcase)
|
183
|
-
require "dynamoid/adapter_plugin/#{Dynamoid::Config.adapter}"
|
184
|
-
end
|
185
|
-
|
186
179
|
Dynamoid::AdapterPlugin.const_get(Dynamoid::Config.adapter.camelcase)
|
187
180
|
end
|
188
181
|
end
|
@@ -3,6 +3,7 @@
|
|
3
3
|
require_relative 'aws_sdk_v3/query'
|
4
4
|
require_relative 'aws_sdk_v3/scan'
|
5
5
|
require_relative 'aws_sdk_v3/create_table'
|
6
|
+
require_relative 'aws_sdk_v3/batch_get_item'
|
6
7
|
require_relative 'aws_sdk_v3/item_updater'
|
7
8
|
require_relative 'aws_sdk_v3/table'
|
8
9
|
require_relative 'aws_sdk_v3/until_past_table_status'
|
@@ -22,8 +23,6 @@ module Dynamoid
|
|
22
23
|
range_eq: 'EQ'
|
23
24
|
}.freeze
|
24
25
|
|
25
|
-
# Don't implement NULL and NOT_NULL because it doesn't make seanse -
|
26
|
-
# we declare schema in models
|
27
26
|
FIELD_MAP = {
|
28
27
|
eq: 'EQ',
|
29
28
|
ne: 'NE',
|
@@ -35,7 +34,9 @@ module Dynamoid
|
|
35
34
|
between: 'BETWEEN',
|
36
35
|
in: 'IN',
|
37
36
|
contains: 'CONTAINS',
|
38
|
-
not_contains: 'NOT_CONTAINS'
|
37
|
+
not_contains: 'NOT_CONTAINS',
|
38
|
+
null: 'NULL',
|
39
|
+
not_null: 'NOT_NULL',
|
39
40
|
}.freeze
|
40
41
|
HASH_KEY = 'HASH'
|
41
42
|
RANGE_KEY = 'RANGE'
|
@@ -183,65 +184,11 @@ module Dynamoid
|
|
183
184
|
# @since 1.0.0
|
184
185
|
#
|
185
186
|
# @todo: Provide support for passing options to underlying batch_get_item
|
186
|
-
def batch_get_item(
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
ret = Hash.new([].freeze) # Default for tables where no rows are returned
|
191
|
-
|
192
|
-
table_ids.each do |t, ids|
|
193
|
-
next if ids.blank?
|
194
|
-
|
195
|
-
ids = Array(ids).dup
|
196
|
-
tbl = describe_table(t)
|
197
|
-
hk = tbl.hash_key.to_s
|
198
|
-
rng = tbl.range_key.to_s
|
199
|
-
|
200
|
-
while ids.present?
|
201
|
-
batch = ids.shift(Dynamoid::Config.batch_size)
|
202
|
-
|
203
|
-
request_items = Hash.new { |h, k| h[k] = [] }
|
204
|
-
|
205
|
-
keys = if rng.present?
|
206
|
-
Array(batch).map do |h, r|
|
207
|
-
{ hk => h, rng => r }
|
208
|
-
end
|
209
|
-
else
|
210
|
-
Array(batch).map do |id|
|
211
|
-
{ hk => id }
|
212
|
-
end
|
213
|
-
end
|
214
|
-
|
215
|
-
request_items[t] = {
|
216
|
-
keys: keys,
|
217
|
-
consistent_read: options[:consistent_read]
|
218
|
-
}
|
219
|
-
|
220
|
-
results = client.batch_get_item(
|
221
|
-
request_items: request_items
|
222
|
-
)
|
223
|
-
|
224
|
-
if block_given?
|
225
|
-
batch_results = Hash.new([].freeze)
|
226
|
-
|
227
|
-
results.data[:responses].each do |table, rows|
|
228
|
-
batch_results[table] += rows.collect { |r| result_item_to_hash(r) }
|
229
|
-
end
|
230
|
-
|
231
|
-
yield(batch_results, results.unprocessed_keys.present?)
|
232
|
-
else
|
233
|
-
results.data[:responses].each do |table, rows|
|
234
|
-
ret[table] += rows.collect { |r| result_item_to_hash(r) }
|
235
|
-
end
|
236
|
-
end
|
237
|
-
|
238
|
-
if results.unprocessed_keys.present?
|
239
|
-
ids += results.unprocessed_keys[t].keys.map { |h| h[hk] }
|
240
|
-
end
|
241
|
-
end
|
187
|
+
def batch_get_item(table_names_with_ids, options = {}, &block)
|
188
|
+
tables_with_ids = table_names_with_ids.transform_keys do |name|
|
189
|
+
describe_table(name)
|
242
190
|
end
|
243
|
-
|
244
|
-
ret unless block_given?
|
191
|
+
BatchGetItem.new(client, tables_with_ids, options).call(&block)
|
245
192
|
end
|
246
193
|
|
247
194
|
# Delete many items at once from DynamoDB. More efficient than delete each item individually.
|
@@ -356,9 +303,14 @@ module Dynamoid
|
|
356
303
|
# @since 1.0.0
|
357
304
|
def delete_table(table_name, options = {})
|
358
305
|
resp = client.delete_table(table_name: table_name)
|
359
|
-
|
360
|
-
|
361
|
-
|
306
|
+
|
307
|
+
if options[:sync]
|
308
|
+
status = PARSE_TABLE_STATUS.call(resp, :table_description)
|
309
|
+
if status == TABLE_STATUSES[:deleting]
|
310
|
+
UntilPastTableStatus.new(client, table_name, :deleting).call
|
311
|
+
end
|
312
|
+
end
|
313
|
+
|
362
314
|
table_cache.delete(table_name)
|
363
315
|
rescue Aws::DynamoDB::Errors::ResourceInUseException => e
|
364
316
|
Dynamoid.logger.error "Table #{table_name} cannot be deleted as it is in use"
|
@@ -619,23 +571,27 @@ module Dynamoid
|
|
619
571
|
# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
|
620
572
|
# @params [String] operator: value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
|
621
573
|
# @params [Object] value: scalar value or array/set
|
622
|
-
def attribute_value_list(operator, value)
|
623
|
-
self.class.attribute_value_list(operator, value)
|
624
|
-
end
|
625
|
-
|
626
574
|
def self.attribute_value_list(operator, value)
|
627
575
|
# For BETWEEN and IN operators we should keep value as is (it should be already an array)
|
576
|
+
# NULL and NOT_NULL require absence of attribute list
|
628
577
|
# For all the other operators we wrap the value with array
|
578
|
+
# https://docs.aws.amazon.com/en_us/amazondynamodb/latest/developerguide/LegacyConditionalParameters.Conditions.html
|
629
579
|
if %w[BETWEEN IN].include?(operator)
|
630
580
|
[value].flatten
|
581
|
+
elsif %w[NULL NOT_NULL].include?(operator)
|
582
|
+
nil
|
631
583
|
else
|
632
584
|
[value]
|
633
585
|
end
|
634
586
|
end
|
635
587
|
|
636
588
|
def sanitize_item(attributes)
|
589
|
+
config_value = Dynamoid.config.store_attribute_with_nil_value
|
590
|
+
store_attribute_with_nil_value = config_value.nil? ? false : !!config_value
|
591
|
+
|
637
592
|
attributes.reject do |_, v|
|
638
|
-
|
593
|
+
((v.is_a?(Set) || v.is_a?(String)) && v.empty?) ||
|
594
|
+
(!store_attribute_with_nil_value && v.nil?)
|
639
595
|
end.transform_values do |v|
|
640
596
|
v.is_a?(Hash) ? v.stringify_keys : v
|
641
597
|
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
module Dynamoid
|
2
|
+
module AdapterPlugin
|
3
|
+
class AwsSdkV3
|
4
|
+
# Documentation
|
5
|
+
# https://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
|
6
|
+
class BatchGetItem
|
7
|
+
attr_reader :client, :tables_with_ids, :options
|
8
|
+
|
9
|
+
def initialize(client, tables_with_ids, options = {})
|
10
|
+
@client = client
|
11
|
+
@tables_with_ids = tables_with_ids
|
12
|
+
@options = options
|
13
|
+
end
|
14
|
+
|
15
|
+
def call
|
16
|
+
results = {}
|
17
|
+
|
18
|
+
tables_with_ids.each do |table, ids|
|
19
|
+
if ids.blank?
|
20
|
+
results[table.name] = []
|
21
|
+
next
|
22
|
+
end
|
23
|
+
|
24
|
+
ids = Array(ids).dup
|
25
|
+
|
26
|
+
while ids.present?
|
27
|
+
batch = ids.shift(Dynamoid::Config.batch_size)
|
28
|
+
request = build_request(table, batch)
|
29
|
+
api_response = client.batch_get_item(request)
|
30
|
+
response = Response.new(api_response)
|
31
|
+
|
32
|
+
if block_given?
|
33
|
+
# return batch items as a result
|
34
|
+
batch_results = Hash.new([].freeze)
|
35
|
+
batch_results.update(response.items_grouped_by_table)
|
36
|
+
|
37
|
+
yield(batch_results, response.successful_partially?)
|
38
|
+
else
|
39
|
+
# collect all the batches to return at the end
|
40
|
+
results.update(response.items_grouped_by_table) { |_, its1, its2| its1 + its2 }
|
41
|
+
end
|
42
|
+
|
43
|
+
if response.successful_partially?
|
44
|
+
ids += response.unprocessed_ids(table)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
results unless block_given?
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def build_request(table, ids)
|
55
|
+
ids = Array(ids)
|
56
|
+
|
57
|
+
keys = if table.range_key.nil?
|
58
|
+
ids.map { |hk| { table.hash_key => hk } }
|
59
|
+
else
|
60
|
+
ids.map { |hk, rk| { table.hash_key => hk, table.range_key => rk } }
|
61
|
+
end
|
62
|
+
|
63
|
+
{
|
64
|
+
request_items: {
|
65
|
+
table.name => {
|
66
|
+
keys: keys,
|
67
|
+
consistent_read: options[:consistent_read]
|
68
|
+
}
|
69
|
+
}
|
70
|
+
}
|
71
|
+
end
|
72
|
+
|
73
|
+
# Helper class to work with response
|
74
|
+
class Response
|
75
|
+
def initialize(api_response)
|
76
|
+
@api_response = api_response
|
77
|
+
end
|
78
|
+
|
79
|
+
def successful_partially?
|
80
|
+
@api_response.unprocessed_keys.present?
|
81
|
+
end
|
82
|
+
|
83
|
+
def unprocessed_ids(table)
|
84
|
+
# unprocessed_keys Hash contains as values instances of
|
85
|
+
# Aws::DynamoDB::Types::KeysAndAttributes
|
86
|
+
@api_response.unprocessed_keys[table.name].keys.map { |h| h[table.hash_key.to_s] }
|
87
|
+
end
|
88
|
+
|
89
|
+
def items_grouped_by_table
|
90
|
+
# data[:responses] is a Hash[table_name -> items]
|
91
|
+
@api_response.data[:responses].transform_values do |items|
|
92
|
+
items.map(&method(:item_to_hash))
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
|
98
|
+
def item_to_hash(item)
|
99
|
+
item.symbolize_keys
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -62,11 +62,16 @@ module Dynamoid
|
|
62
62
|
end
|
63
63
|
resp = client.create_table(client_opts)
|
64
64
|
options[:sync] = true if !options.key?(:sync) && ls_indexes.present? || gs_indexes.present?
|
65
|
-
|
66
|
-
|
67
|
-
|
65
|
+
|
66
|
+
if options[:sync]
|
67
|
+
status = PARSE_TABLE_STATUS.call(resp, :table_description)
|
68
|
+
if status == TABLE_STATUSES[:creating]
|
69
|
+
UntilPastTableStatus.new(client, table_name, :creating).call
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
68
73
|
# Response to original create_table, which, if options[:sync]
|
69
|
-
#
|
74
|
+
# may have an outdated table_description.table_status of "CREATING"
|
70
75
|
resp
|
71
76
|
end
|
72
77
|
|
@@ -11,6 +11,7 @@ module Dynamoid
|
|
11
11
|
OPTIONS_KEYS = %i[
|
12
12
|
limit hash_key hash_value range_key consistent_read scan_index_forward
|
13
13
|
select index_name batch_size exclusive_start_key record_limit scan_limit
|
14
|
+
project
|
14
15
|
].freeze
|
15
16
|
|
16
17
|
attr_reader :client, :table, :options, :conditions
|
@@ -63,10 +64,11 @@ module Dynamoid
|
|
63
64
|
batch_size = options[:batch_size]
|
64
65
|
limit = [record_limit, scan_limit, batch_size].compact.min
|
65
66
|
|
66
|
-
request[:limit]
|
67
|
-
request[:table_name]
|
68
|
-
request[:key_conditions]
|
69
|
-
request[:query_filter]
|
67
|
+
request[:limit] = limit if limit
|
68
|
+
request[:table_name] = table.name
|
69
|
+
request[:key_conditions] = key_conditions
|
70
|
+
request[:query_filter] = query_filter
|
71
|
+
request[:attributes_to_get] = attributes_to_get
|
70
72
|
|
71
73
|
request
|
72
74
|
end
|
@@ -117,6 +119,11 @@ module Dynamoid
|
|
117
119
|
result
|
118
120
|
end
|
119
121
|
end
|
122
|
+
|
123
|
+
def attributes_to_get
|
124
|
+
return if options[:project].nil?
|
125
|
+
options[:project].map(&:to_s)
|
126
|
+
end
|
120
127
|
end
|
121
128
|
end
|
122
129
|
end
|
@@ -54,9 +54,10 @@ module Dynamoid
|
|
54
54
|
batch_size = options[:batch_size]
|
55
55
|
limit = [record_limit, scan_limit, batch_size].compact.min
|
56
56
|
|
57
|
-
request[:limit]
|
58
|
-
request[:table_name]
|
59
|
-
request[:scan_filter]
|
57
|
+
request[:limit] = limit if limit
|
58
|
+
request[:table_name] = table.name
|
59
|
+
request[:scan_filter] = scan_filter
|
60
|
+
request[:attributes_to_get] = attributes_to_get
|
60
61
|
|
61
62
|
request
|
62
63
|
end
|
@@ -75,10 +76,17 @@ module Dynamoid
|
|
75
76
|
comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
|
76
77
|
attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
|
77
78
|
}
|
79
|
+
# nil means operator doesn't require attribute value list
|
80
|
+
conditions.delete(:attribute_value_list) if conditions[:attribute_value_list].nil?
|
78
81
|
result[attr] = condition
|
79
82
|
result
|
80
83
|
end
|
81
84
|
end
|
85
|
+
|
86
|
+
def attributes_to_get
|
87
|
+
return if options[:project].nil?
|
88
|
+
options[:project].map(&:to_s)
|
89
|
+
end
|
82
90
|
end
|
83
91
|
end
|
84
92
|
end
|