dynamoid 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,11 +1,9 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activemodel', '~> 4.2.0'
8
- gem 'nokogiri', '~> 1.6.8'
9
- gem 'pry-byebug', platforms: :ruby
5
+ gem "pry-byebug", platforms: :ruby
6
+ gem "activemodel", "~> 4.2.0"
7
+ gem "nokogiri", "~> 1.6.8"
10
8
 
11
- gemspec path: '../'
9
+ gemspec path: "../"
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activemodel', '~> 5.0.0'
8
- gem 'pry-byebug', platforms: :ruby
5
+ gem "pry-byebug", platforms: :ruby
6
+ gem "activemodel", "~> 5.0.0"
9
7
 
10
- gemspec path: '../'
8
+ gemspec path: "../"
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activemodel', '~> 5.1.0'
8
- gem 'pry-byebug', platforms: :ruby
5
+ gem "pry-byebug", platforms: :ruby
6
+ gem "activemodel", "~> 5.1.0"
9
7
 
10
- gemspec path: '../'
8
+ gemspec path: "../"
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activemodel', '~> 5.2.0'
8
- gem 'pry-byebug', platforms: :ruby
5
+ gem "pry-byebug", platforms: :ruby
6
+ gem "activemodel", "~> 5.2.0"
9
7
 
10
- gemspec path: '../'
8
+ gemspec path: "../"
@@ -0,0 +1,8 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "pry-byebug", platforms: :ruby
6
+ gem "activemodel", "6.0.0"
7
+
8
+ gemspec path: "../"
data/lib/dynamoid.rb CHANGED
@@ -30,6 +30,7 @@ require 'dynamoid/criteria'
30
30
  require 'dynamoid/finders'
31
31
  require 'dynamoid/identity_map'
32
32
  require 'dynamoid/config'
33
+ require 'dynamoid/loadable'
33
34
  require 'dynamoid/components'
34
35
  require 'dynamoid/document'
35
36
  require 'dynamoid/adapter'
@@ -3,6 +3,7 @@
3
3
  # require only 'concurrent/atom' once this issue is resolved:
4
4
  # https://github.com/ruby-concurrency/concurrent-ruby/pull/377
5
5
  require 'concurrent'
6
+ require "dynamoid/adapter_plugin/aws_sdk_v3"
6
7
 
7
8
  # encoding: utf-8
8
9
  module Dynamoid
@@ -29,7 +30,7 @@ module Dynamoid
29
30
  def adapter
30
31
  unless @adapter_.value
31
32
  adapter = self.class.adapter_plugin_class.new
32
- adapter.connect! if adapter.respond_to?(:connect!)
33
+ adapter.connect!
33
34
  @adapter_.compare_and_set(nil, adapter)
34
35
  clear_cache!
35
36
  end
@@ -142,11 +143,7 @@ module Dynamoid
142
143
  #
143
144
  # @since 0.2.0
144
145
  define_method(m) do |*args, &blk|
145
- if blk.present?
146
- benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
147
- else
148
- benchmark(m.to_s, *args) { adapter.send(m, *args) }
149
- end
146
+ benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
150
147
  end
151
148
  end
152
149
 
@@ -179,10 +176,6 @@ module Dynamoid
179
176
  end
180
177
 
181
178
  def self.adapter_plugin_class
182
- unless Dynamoid.const_defined?(:AdapterPlugin) && Dynamoid::AdapterPlugin.const_defined?(Dynamoid::Config.adapter.camelcase)
183
- require "dynamoid/adapter_plugin/#{Dynamoid::Config.adapter}"
184
- end
185
-
186
179
  Dynamoid::AdapterPlugin.const_get(Dynamoid::Config.adapter.camelcase)
187
180
  end
188
181
  end
@@ -3,6 +3,7 @@
3
3
  require_relative 'aws_sdk_v3/query'
4
4
  require_relative 'aws_sdk_v3/scan'
5
5
  require_relative 'aws_sdk_v3/create_table'
6
+ require_relative 'aws_sdk_v3/batch_get_item'
6
7
  require_relative 'aws_sdk_v3/item_updater'
7
8
  require_relative 'aws_sdk_v3/table'
8
9
  require_relative 'aws_sdk_v3/until_past_table_status'
@@ -22,8 +23,6 @@ module Dynamoid
22
23
  range_eq: 'EQ'
23
24
  }.freeze
24
25
 
25
- # Don't implement NULL and NOT_NULL because it doesn't make seanse -
26
- # we declare schema in models
27
26
  FIELD_MAP = {
28
27
  eq: 'EQ',
29
28
  ne: 'NE',
@@ -35,7 +34,9 @@ module Dynamoid
35
34
  between: 'BETWEEN',
36
35
  in: 'IN',
37
36
  contains: 'CONTAINS',
38
- not_contains: 'NOT_CONTAINS'
37
+ not_contains: 'NOT_CONTAINS',
38
+ null: 'NULL',
39
+ not_null: 'NOT_NULL',
39
40
  }.freeze
40
41
  HASH_KEY = 'HASH'
41
42
  RANGE_KEY = 'RANGE'
@@ -183,65 +184,11 @@ module Dynamoid
183
184
  # @since 1.0.0
184
185
  #
185
186
  # @todo: Provide support for passing options to underlying batch_get_item
186
- def batch_get_item(table_ids, options = {})
187
- request_items = Hash.new { |h, k| h[k] = [] }
188
- return request_items if table_ids.all? { |_k, v| v.blank? }
189
-
190
- ret = Hash.new([].freeze) # Default for tables where no rows are returned
191
-
192
- table_ids.each do |t, ids|
193
- next if ids.blank?
194
-
195
- ids = Array(ids).dup
196
- tbl = describe_table(t)
197
- hk = tbl.hash_key.to_s
198
- rng = tbl.range_key.to_s
199
-
200
- while ids.present?
201
- batch = ids.shift(Dynamoid::Config.batch_size)
202
-
203
- request_items = Hash.new { |h, k| h[k] = [] }
204
-
205
- keys = if rng.present?
206
- Array(batch).map do |h, r|
207
- { hk => h, rng => r }
208
- end
209
- else
210
- Array(batch).map do |id|
211
- { hk => id }
212
- end
213
- end
214
-
215
- request_items[t] = {
216
- keys: keys,
217
- consistent_read: options[:consistent_read]
218
- }
219
-
220
- results = client.batch_get_item(
221
- request_items: request_items
222
- )
223
-
224
- if block_given?
225
- batch_results = Hash.new([].freeze)
226
-
227
- results.data[:responses].each do |table, rows|
228
- batch_results[table] += rows.collect { |r| result_item_to_hash(r) }
229
- end
230
-
231
- yield(batch_results, results.unprocessed_keys.present?)
232
- else
233
- results.data[:responses].each do |table, rows|
234
- ret[table] += rows.collect { |r| result_item_to_hash(r) }
235
- end
236
- end
237
-
238
- if results.unprocessed_keys.present?
239
- ids += results.unprocessed_keys[t].keys.map { |h| h[hk] }
240
- end
241
- end
187
+ def batch_get_item(table_names_with_ids, options = {}, &block)
188
+ tables_with_ids = table_names_with_ids.transform_keys do |name|
189
+ describe_table(name)
242
190
  end
243
-
244
- ret unless block_given?
191
+ BatchGetItem.new(client, tables_with_ids, options).call(&block)
245
192
  end
246
193
 
247
194
  # Delete many items at once from DynamoDB. More efficient than delete each item individually.
@@ -356,9 +303,14 @@ module Dynamoid
356
303
  # @since 1.0.0
357
304
  def delete_table(table_name, options = {})
358
305
  resp = client.delete_table(table_name: table_name)
359
- UntilPastTableStatus.new(table_name, :deleting).call if options[:sync] &&
360
- (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
361
- status == TABLE_STATUSES[:deleting]
306
+
307
+ if options[:sync]
308
+ status = PARSE_TABLE_STATUS.call(resp, :table_description)
309
+ if status == TABLE_STATUSES[:deleting]
310
+ UntilPastTableStatus.new(client, table_name, :deleting).call
311
+ end
312
+ end
313
+
362
314
  table_cache.delete(table_name)
363
315
  rescue Aws::DynamoDB::Errors::ResourceInUseException => e
364
316
  Dynamoid.logger.error "Table #{table_name} cannot be deleted as it is in use"
@@ -619,23 +571,27 @@ module Dynamoid
619
571
  # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
620
572
  # @params [String] operator: value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
621
573
  # @params [Object] value: scalar value or array/set
622
- def attribute_value_list(operator, value)
623
- self.class.attribute_value_list(operator, value)
624
- end
625
-
626
574
  def self.attribute_value_list(operator, value)
627
575
  # For BETWEEN and IN operators we should keep value as is (it should be already an array)
576
+ # NULL and NOT_NULL require absence of attribute list
628
577
  # For all the other operators we wrap the value with array
578
+ # https://docs.aws.amazon.com/en_us/amazondynamodb/latest/developerguide/LegacyConditionalParameters.Conditions.html
629
579
  if %w[BETWEEN IN].include?(operator)
630
580
  [value].flatten
581
+ elsif %w[NULL NOT_NULL].include?(operator)
582
+ nil
631
583
  else
632
584
  [value]
633
585
  end
634
586
  end
635
587
 
636
588
  def sanitize_item(attributes)
589
+ config_value = Dynamoid.config.store_attribute_with_nil_value
590
+ store_attribute_with_nil_value = config_value.nil? ? false : !!config_value
591
+
637
592
  attributes.reject do |_, v|
638
- v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
593
+ ((v.is_a?(Set) || v.is_a?(String)) && v.empty?) ||
594
+ (!store_attribute_with_nil_value && v.nil?)
639
595
  end.transform_values do |v|
640
596
  v.is_a?(Hash) ? v.stringify_keys : v
641
597
  end
@@ -0,0 +1,105 @@
1
+ module Dynamoid
2
+ module AdapterPlugin
3
+ class AwsSdkV3
4
+ # Documentation
5
+ # https://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
6
+ class BatchGetItem
7
+ attr_reader :client, :tables_with_ids, :options
8
+
9
+ def initialize(client, tables_with_ids, options = {})
10
+ @client = client
11
+ @tables_with_ids = tables_with_ids
12
+ @options = options
13
+ end
14
+
15
+ def call
16
+ results = {}
17
+
18
+ tables_with_ids.each do |table, ids|
19
+ if ids.blank?
20
+ results[table.name] = []
21
+ next
22
+ end
23
+
24
+ ids = Array(ids).dup
25
+
26
+ while ids.present?
27
+ batch = ids.shift(Dynamoid::Config.batch_size)
28
+ request = build_request(table, batch)
29
+ api_response = client.batch_get_item(request)
30
+ response = Response.new(api_response)
31
+
32
+ if block_given?
33
+ # return batch items as a result
34
+ batch_results = Hash.new([].freeze)
35
+ batch_results.update(response.items_grouped_by_table)
36
+
37
+ yield(batch_results, response.successful_partially?)
38
+ else
39
+ # collect all the batches to return at the end
40
+ results.update(response.items_grouped_by_table) { |_, its1, its2| its1 + its2 }
41
+ end
42
+
43
+ if response.successful_partially?
44
+ ids += response.unprocessed_ids(table)
45
+ end
46
+ end
47
+ end
48
+
49
+ results unless block_given?
50
+ end
51
+
52
+ private
53
+
54
+ def build_request(table, ids)
55
+ ids = Array(ids)
56
+
57
+ keys = if table.range_key.nil?
58
+ ids.map { |hk| { table.hash_key => hk } }
59
+ else
60
+ ids.map { |hk, rk| { table.hash_key => hk, table.range_key => rk } }
61
+ end
62
+
63
+ {
64
+ request_items: {
65
+ table.name => {
66
+ keys: keys,
67
+ consistent_read: options[:consistent_read]
68
+ }
69
+ }
70
+ }
71
+ end
72
+
73
+ # Helper class to work with response
74
+ class Response
75
+ def initialize(api_response)
76
+ @api_response = api_response
77
+ end
78
+
79
+ def successful_partially?
80
+ @api_response.unprocessed_keys.present?
81
+ end
82
+
83
+ def unprocessed_ids(table)
84
+ # unprocessed_keys Hash contains as values instances of
85
+ # Aws::DynamoDB::Types::KeysAndAttributes
86
+ @api_response.unprocessed_keys[table.name].keys.map { |h| h[table.hash_key.to_s] }
87
+ end
88
+
89
+ def items_grouped_by_table
90
+ # data[:responses] is a Hash[table_name -> items]
91
+ @api_response.data[:responses].transform_values do |items|
92
+ items.map(&method(:item_to_hash))
93
+ end
94
+ end
95
+
96
+ private
97
+
98
+ def item_to_hash(item)
99
+ item.symbolize_keys
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -62,11 +62,16 @@ module Dynamoid
62
62
  end
63
63
  resp = client.create_table(client_opts)
64
64
  options[:sync] = true if !options.key?(:sync) && ls_indexes.present? || gs_indexes.present?
65
- UntilPastTableStatus.new(table_name, :creating).call if options[:sync] &&
66
- (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
67
- status == TABLE_STATUSES[:creating]
65
+
66
+ if options[:sync]
67
+ status = PARSE_TABLE_STATUS.call(resp, :table_description)
68
+ if status == TABLE_STATUSES[:creating]
69
+ UntilPastTableStatus.new(client, table_name, :creating).call
70
+ end
71
+ end
72
+
68
73
  # Response to original create_table, which, if options[:sync]
69
- # may have an outdated table_description.table_status of "CREATING"
74
+ # may have an outdated table_description.table_status of "CREATING"
70
75
  resp
71
76
  end
72
77
 
@@ -11,6 +11,7 @@ module Dynamoid
11
11
  OPTIONS_KEYS = %i[
12
12
  limit hash_key hash_value range_key consistent_read scan_index_forward
13
13
  select index_name batch_size exclusive_start_key record_limit scan_limit
14
+ project
14
15
  ].freeze
15
16
 
16
17
  attr_reader :client, :table, :options, :conditions
@@ -63,10 +64,11 @@ module Dynamoid
63
64
  batch_size = options[:batch_size]
64
65
  limit = [record_limit, scan_limit, batch_size].compact.min
65
66
 
66
- request[:limit] = limit if limit
67
- request[:table_name] = table.name
68
- request[:key_conditions] = key_conditions
69
- request[:query_filter] = query_filter
67
+ request[:limit] = limit if limit
68
+ request[:table_name] = table.name
69
+ request[:key_conditions] = key_conditions
70
+ request[:query_filter] = query_filter
71
+ request[:attributes_to_get] = attributes_to_get
70
72
 
71
73
  request
72
74
  end
@@ -117,6 +119,11 @@ module Dynamoid
117
119
  result
118
120
  end
119
121
  end
122
+
123
+ def attributes_to_get
124
+ return if options[:project].nil?
125
+ options[:project].map(&:to_s)
126
+ end
120
127
  end
121
128
  end
122
129
  end
@@ -54,9 +54,10 @@ module Dynamoid
54
54
  batch_size = options[:batch_size]
55
55
  limit = [record_limit, scan_limit, batch_size].compact.min
56
56
 
57
- request[:limit] = limit if limit
58
- request[:table_name] = table.name
59
- request[:scan_filter] = scan_filter
57
+ request[:limit] = limit if limit
58
+ request[:table_name] = table.name
59
+ request[:scan_filter] = scan_filter
60
+ request[:attributes_to_get] = attributes_to_get
60
61
 
61
62
  request
62
63
  end
@@ -75,10 +76,17 @@ module Dynamoid
75
76
  comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
76
77
  attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
77
78
  }
79
+ # nil means operator doesn't require attribute value list
80
+ conditions.delete(:attribute_value_list) if conditions[:attribute_value_list].nil?
78
81
  result[attr] = condition
79
82
  result
80
83
  end
81
84
  end
85
+
86
+ def attributes_to_get
87
+ return if options[:project].nil?
88
+ options[:project].map(&:to_s)
89
+ end
82
90
  end
83
91
  end
84
92
  end