dynamoid 3.3.0 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +104 -1
  3. data/README.md +146 -52
  4. data/lib/dynamoid.rb +1 -0
  5. data/lib/dynamoid/adapter.rb +20 -7
  6. data/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb +70 -37
  7. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/batch_get_item.rb +3 -0
  8. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/create_table.rb +20 -12
  9. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/item_updater.rb +5 -4
  10. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/middleware/backoff.rb +2 -2
  11. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/middleware/limit.rb +2 -3
  12. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/middleware/start_key.rb +2 -2
  13. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb +4 -2
  14. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb +4 -2
  15. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/table.rb +1 -0
  16. data/lib/dynamoid/adapter_plugin/aws_sdk_v3/until_past_table_status.rb +2 -1
  17. data/lib/dynamoid/application_time_zone.rb +1 -0
  18. data/lib/dynamoid/associations.rb +182 -19
  19. data/lib/dynamoid/associations/association.rb +10 -2
  20. data/lib/dynamoid/associations/belongs_to.rb +2 -1
  21. data/lib/dynamoid/associations/has_and_belongs_to_many.rb +2 -1
  22. data/lib/dynamoid/associations/has_many.rb +2 -1
  23. data/lib/dynamoid/associations/has_one.rb +2 -1
  24. data/lib/dynamoid/associations/many_association.rb +68 -23
  25. data/lib/dynamoid/associations/single_association.rb +31 -4
  26. data/lib/dynamoid/components.rb +2 -0
  27. data/lib/dynamoid/config.rb +15 -3
  28. data/lib/dynamoid/config/backoff_strategies/constant_backoff.rb +1 -0
  29. data/lib/dynamoid/config/backoff_strategies/exponential_backoff.rb +1 -0
  30. data/lib/dynamoid/config/options.rb +1 -0
  31. data/lib/dynamoid/criteria.rb +9 -1
  32. data/lib/dynamoid/criteria/chain.rb +421 -46
  33. data/lib/dynamoid/criteria/ignored_conditions_detector.rb +3 -3
  34. data/lib/dynamoid/criteria/key_fields_detector.rb +31 -10
  35. data/lib/dynamoid/criteria/nonexistent_fields_detector.rb +3 -2
  36. data/lib/dynamoid/criteria/overwritten_conditions_detector.rb +1 -1
  37. data/lib/dynamoid/dirty.rb +119 -64
  38. data/lib/dynamoid/document.rb +133 -46
  39. data/lib/dynamoid/dumping.rb +9 -0
  40. data/lib/dynamoid/dynamodb_time_zone.rb +1 -0
  41. data/lib/dynamoid/errors.rb +2 -0
  42. data/lib/dynamoid/fields.rb +251 -39
  43. data/lib/dynamoid/fields/declare.rb +86 -0
  44. data/lib/dynamoid/finders.rb +69 -32
  45. data/lib/dynamoid/identity_map.rb +6 -0
  46. data/lib/dynamoid/indexes.rb +86 -17
  47. data/lib/dynamoid/loadable.rb +2 -2
  48. data/lib/dynamoid/log/formatter.rb +26 -0
  49. data/lib/dynamoid/middleware/identity_map.rb +1 -0
  50. data/lib/dynamoid/persistence.rb +502 -104
  51. data/lib/dynamoid/persistence/import.rb +2 -1
  52. data/lib/dynamoid/persistence/save.rb +1 -0
  53. data/lib/dynamoid/persistence/update_fields.rb +5 -2
  54. data/lib/dynamoid/persistence/update_validations.rb +18 -0
  55. data/lib/dynamoid/persistence/upsert.rb +5 -3
  56. data/lib/dynamoid/primary_key_type_mapping.rb +1 -0
  57. data/lib/dynamoid/railtie.rb +1 -0
  58. data/lib/dynamoid/tasks.rb +3 -1
  59. data/lib/dynamoid/tasks/database.rb +1 -0
  60. data/lib/dynamoid/type_casting.rb +12 -2
  61. data/lib/dynamoid/undumping.rb +8 -0
  62. data/lib/dynamoid/validations.rb +6 -1
  63. data/lib/dynamoid/version.rb +1 -1
  64. metadata +48 -75
  65. data/.coveralls.yml +0 -1
  66. data/.document +0 -5
  67. data/.gitignore +0 -74
  68. data/.rspec +0 -2
  69. data/.rubocop.yml +0 -71
  70. data/.rubocop_todo.yml +0 -55
  71. data/.travis.yml +0 -44
  72. data/Appraisals +0 -22
  73. data/Gemfile +0 -8
  74. data/Rakefile +0 -46
  75. data/Vagrantfile +0 -29
  76. data/docker-compose.yml +0 -7
  77. data/dynamoid.gemspec +0 -57
  78. data/gemfiles/rails_4_2.gemfile +0 -9
  79. data/gemfiles/rails_5_0.gemfile +0 -8
  80. data/gemfiles/rails_5_1.gemfile +0 -8
  81. data/gemfiles/rails_5_2.gemfile +0 -8
  82. data/gemfiles/rails_6_0.gemfile +0 -8
@@ -1,12 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Dynamoid #:nodoc:
3
+ module Dynamoid
4
4
  module Associations
5
5
  module SingleAssociation
6
6
  include Association
7
7
 
8
8
  delegate :class, to: :target
9
9
 
10
+ # @private
10
11
  def setter(object)
11
12
  if object.nil?
12
13
  delete
@@ -19,16 +20,37 @@ module Dynamoid #:nodoc:
19
20
  object
20
21
  end
21
22
 
23
+ # Delete a model from the association.
24
+ #
25
+ # post.logo.delete # => nil
26
+ #
27
+ # Saves both models immediately - a source model and a target one so any
28
+ # unsaved changes will be saved. Doesn't delete an associated model from
29
+ # DynamoDB.
22
30
  def delete
23
- target.send(target_association).disassociate(source.hash_key) if target && target_association
31
+ disassociate_source
24
32
  disassociate
25
33
  target
26
34
  end
27
35
 
36
+ # Create a new instance of the target class, persist it and associate.
37
+ #
38
+ # post.logo.create!(hight: 50, width: 90)
39
+ #
40
+ # If the creation fails an exception will be raised.
41
+ #
42
+ # @param attributes [Hash] attributes of a model to create
43
+ # @return [Dynamoid::Document] created model
28
44
  def create!(attributes = {})
29
45
  setter(target_class.create!(attributes))
30
46
  end
31
47
 
48
+ # Create a new instance of the target class, persist it and associate.
49
+ #
50
+ # post.logo.create(hight: 50, width: 90)
51
+ #
52
+ # @param attributes [Hash] attributes of a model to create
53
+ # @return [Dynamoid::Document] created model
32
54
  def create(attributes = {})
33
55
  setter(target_class.create(attributes))
34
56
  end
@@ -44,6 +66,7 @@ module Dynamoid #:nodoc:
44
66
 
45
67
  # Delegate methods we don't find directly to the target.
46
68
  #
69
+ # @private
47
70
  # @since 0.2.0
48
71
  def method_missing(method, *args)
49
72
  if target.respond_to?(method)
@@ -53,21 +76,25 @@ module Dynamoid #:nodoc:
53
76
  end
54
77
  end
55
78
 
79
+ # @private
56
80
  def nil?
57
81
  target.nil?
58
82
  end
59
83
 
84
+ # @private
60
85
  def empty?
61
86
  # This is needed to that ActiveSupport's #blank? and #present?
62
87
  # methods work as expected for SingleAssociations.
63
88
  target.nil?
64
89
  end
65
90
 
91
+ # @private
66
92
  def associate(hash_key)
67
- target.send(target_association).disassociate(source.hash_key) if target && target_association
93
+ disassociate_source
68
94
  source.update_attribute(source_attribute, Set[hash_key])
69
95
  end
70
96
 
97
+ # @private
71
98
  def disassociate(_hash_key = nil)
72
99
  source.update_attribute(source_attribute, nil)
73
100
  end
@@ -82,7 +109,7 @@ module Dynamoid #:nodoc:
82
109
  def find_target
83
110
  return if source_ids.empty?
84
111
 
85
- target_class.find(source_ids.first)
112
+ target_class.find(source_ids.first, raise_error: false)
86
113
  end
87
114
 
88
115
  def target=(object)
@@ -3,6 +3,7 @@
3
3
  module Dynamoid
4
4
  # All modules that a Document is composed of are defined in this
5
5
  # module, to keep the document class from getting too cluttered.
6
+ # @private
6
7
  module Components
7
8
  extend ActiveSupport::Concern
8
9
 
@@ -14,6 +15,7 @@ module Dynamoid
14
15
 
15
16
  before_create :set_created_at
16
17
  before_save :set_updated_at
18
+ before_save :set_expires_field
17
19
  after_initialize :set_inheritance_field
18
20
  end
19
21
 
@@ -2,14 +2,23 @@
2
2
 
3
3
  require 'uri'
4
4
  require 'logger'
5
- require 'null_logger'
6
5
  require 'dynamoid/config/options'
7
6
  require 'dynamoid/config/backoff_strategies/constant_backoff'
8
7
  require 'dynamoid/config/backoff_strategies/exponential_backoff'
9
8
 
10
9
  module Dynamoid
11
10
  # Contains all the basic configuration information required for Dynamoid: both sensible defaults and required fields.
11
+ # @private
12
12
  module Config
13
+ # @since 3.3.1
14
+ DEFAULT_NAMESPACE = if defined?(Rails)
15
+ klass = Rails.application.class
16
+ app_name = Rails::VERSION::MAJOR >= 6 ? klass.module_parent_name : klass.parent_name
17
+ "dynamoid_#{app_name}_#{Rails.env}"
18
+ else
19
+ 'dynamoid'
20
+ end
21
+
13
22
  extend self
14
23
 
15
24
  extend Options
@@ -17,11 +26,13 @@ module Dynamoid
17
26
 
18
27
  # All the default options.
19
28
  option :adapter, default: 'aws_sdk_v3'
20
- option :namespace, default: defined?(Rails) ? "dynamoid_#{Rails.application.class.parent_name}_#{Rails.env}" : 'dynamoid'
29
+ option :namespace, default: DEFAULT_NAMESPACE
21
30
  option :access_key, default: nil
22
31
  option :secret_key, default: nil
32
+ option :credentials, default: nil
23
33
  option :region, default: nil
24
34
  option :batch_size, default: 100
35
+ option :capacity_mode, default: nil
25
36
  option :read_capacity, default: 100
26
37
  option :write_capacity, default: 20
27
38
  option :warn_on_scan, default: true
@@ -43,6 +54,7 @@ module Dynamoid
43
54
  constant: BackoffStrategies::ConstantBackoff,
44
55
  exponential: BackoffStrategies::ExponentialBackoff
45
56
  }
57
+ option :log_formatter, default: nil
46
58
  option :http_continue_timeout, default: nil # specify if you'd like to overwrite Aws Configure - default: 1
47
59
  option :http_idle_timeout, default: nil # - default: 5
48
60
  option :http_open_timeout, default: nil # - default: 15
@@ -67,7 +79,7 @@ module Dynamoid
67
79
  # @since 0.2.0
68
80
  def logger=(logger)
69
81
  case logger
70
- when false, nil then @logger = NullLogger.new
82
+ when false, nil then @logger = ::Logger.new(nil)
71
83
  when true then @logger = default_logger
72
84
  else
73
85
  @logger = logger if logger.respond_to?(:info)
@@ -2,6 +2,7 @@
2
2
 
3
3
  module Dynamoid
4
4
  module Config
5
+ # @private
5
6
  module BackoffStrategies
6
7
  class ConstantBackoff
7
8
  def self.call(sec = 1)
@@ -2,6 +2,7 @@
2
2
 
3
3
  module Dynamoid
4
4
  module Config
5
+ # @private
5
6
  module BackoffStrategies
6
7
  # Truncated binary exponential backoff algorithm
7
8
  # See https://en.wikipedia.org/wiki/Exponential_backoff
@@ -4,6 +4,7 @@
4
4
  module Dynamoid
5
5
  module Config
6
6
  # Encapsulates logic for setting options.
7
+ # @private
7
8
  module Options
8
9
  # Get the defaults or initialize a new empty hash.
9
10
  #
@@ -7,13 +7,21 @@ module Dynamoid
7
7
  module Criteria
8
8
  extend ActiveSupport::Concern
9
9
 
10
+ # @private
10
11
  module ClassMethods
11
- %i[where all first last each record_limit scan_limit batch start scan_index_forward find_by_pages project].each do |meth|
12
+ %i[where consistent all first last delete_all destroy_all each record_limit scan_limit batch start scan_index_forward find_by_pages project pluck].each do |meth|
12
13
  # Return a criteria chain in response to a method that will begin or end a chain. For more information,
13
14
  # see Dynamoid::Criteria::Chain.
14
15
  #
15
16
  # @since 0.2.0
16
17
  define_method(meth) do |*args, &blk|
18
+ # Don't use keywork arguments delegating (with **kw). It works in
19
+ # different way in different Ruby versions: <= 2.6, 2.7, 3.0 and in some
20
+ # future 3.x versions. Providing that there are no downstream methods
21
+ # with keyword arguments in Chain.
22
+ #
23
+ # https://eregon.me/blog/2019/11/10/the-delegation-challenge-of-ruby27.html
24
+
17
25
  chain = Dynamoid::Criteria::Chain.new(self)
18
26
  if args
19
27
  chain.send(meth, *args, &blk)
@@ -22,26 +22,74 @@ module Dynamoid
22
22
  @consistent_read = false
23
23
  @scan_index_forward = true
24
24
 
25
- # Honor STI and :type field if it presents
26
- type = @source.inheritance_field
27
- if @source.attributes.key?(type)
28
- @query[:"#{type}.in"] = @source.deep_subclasses.map(&:name) << @source.name
29
- end
30
-
31
25
  # we should re-initialize keys detector every time we change query
32
26
  @key_fields_detector = KeyFieldsDetector.new(@query, @source)
33
27
  end
34
28
 
35
- # The workhorse method of the criteria chain. Each key in the passed in hash will become another criteria that the
36
- # ultimate query must match. A key can either be a symbol or a string, and should be an attribute name or
37
- # an attribute name with a range operator.
29
+ # Returns a chain which is a result of filtering current chain with the specified conditions.
30
+ #
31
+ # It accepts conditions in the form of a hash.
32
+ #
33
+ # Post.where(links_count: 2)
34
+ #
35
+ # A key could be either string or symbol.
36
+ #
37
+ # In order to express conditions other than equality predicates could be used.
38
+ # Predicate should be added to an attribute name to form a key +'created_at.gt' => Date.yesterday+
39
+ #
40
+ # Currently supported following predicates:
41
+ # - +gt+ - greater than
42
+ # - +gte+ - greater or equal
43
+ # - +lt+ - less than
44
+ # - +lte+ - less or equal
45
+ # - +ne+ - not equal
46
+ # - +between+ - an attribute value is greater than the first value and less than the second value
47
+ # - +in+ - check an attribute in a list of values
48
+ # - +begins_with+ - check for a prefix in string
49
+ # - +contains+ - check substring or value in a set or array
50
+ # - +not_contains+ - check for absence of substring or a value in set or array
51
+ # - +null+ - attribute doesn't exists in an item
52
+ # - +not_null+ - attribute exists in an item
53
+ #
54
+ # All the predicates match operators supported by DynamoDB's
55
+ # {ComparisonOperator}[https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html#DDB-Type-Condition-ComparisonOperator]
56
+ #
57
+ # Post.where('size.gt' => 1000)
58
+ # Post.where('size.gte' => 1000)
59
+ # Post.where('size.lt' => 35000)
60
+ # Post.where('size.lte' => 35000)
61
+ # Post.where('author.ne' => 'John Doe')
62
+ # Post.where('created_at.between' => [Time.now - 3600, Time.now])
63
+ # Post.where('category.in' => ['tech', 'fashion'])
64
+ # Post.where('title.begins_with' => 'How long')
65
+ # Post.where('tags.contains' => 'Ruby')
66
+ # Post.where('tags.not_contains' => 'Ruby on Rails')
67
+ # Post.where('legacy_attribute.null' => true)
68
+ # Post.where('optional_attribute.not_null' => true)
69
+ #
70
+ # There are some limitations for a sort key. Only following predicates
71
+ # are supported - +gt+, +gte+, +lt+, +lte+, +between+, +begins_with+.
72
+ #
73
+ # +where+ without argument will return the current chain.
74
+ #
75
+ # Multiple calls can be chained together and conditions will be merged:
76
+ #
77
+ # Post.where('size.gt' => 1000).where('title' => 'some title')
78
+ #
79
+ # It's equivalent to:
80
+ #
81
+ # Post.where('size.gt' => 1000, 'title' => 'some title')
82
+ #
83
+ # But only one condition can be specified for a certain attribute. The
84
+ # last specified condition will override all the others. Only condition
85
+ # 'size.lt' => 200 will be used in following examples:
38
86
  #
39
- # @example A simple criteria
40
- # where(:name => 'Josh')
87
+ # Post.where('size.gt' => 100, 'size.lt' => 200)
88
+ # Post.where('size.gt' => 100).where('size.lt' => 200)
41
89
  #
42
- # @example A more complicated criteria
43
- # where(:name => 'Josh', 'created_at.gt' => DateTime.now - 1.day)
90
+ # Internally +where+ performs either +Scan+ or +Query+ operation.
44
91
  #
92
+ # @return [Dynamoid::Criteria::Chain]
45
93
  # @since 0.2.0
46
94
  def where(args)
47
95
  detector = IgnoredConditionsDetector.new(args)
@@ -62,11 +110,18 @@ module Dynamoid
62
110
  query.update(args.symbolize_keys)
63
111
 
64
112
  # we should re-initialize keys detector every time we change query
65
- @key_fields_detector = KeyFieldsDetector.new(@query, @source)
113
+ @key_fields_detector = KeyFieldsDetector.new(@query, @source, forced_index_name: @forced_index_name)
66
114
 
67
115
  self
68
116
  end
69
117
 
118
+ # Turns on strongly consistent reads.
119
+ #
120
+ # By default reads are eventually consistent.
121
+ #
122
+ # Post.where('size.gt' => 1000).consistent
123
+ #
124
+ # @return [Dynamoid::Criteria::Chain]
70
125
  def consistent
71
126
  @consistent_read = true
72
127
  self
@@ -74,11 +129,39 @@ module Dynamoid
74
129
 
75
130
  # Returns all the records matching the criteria.
76
131
  #
132
+ # Since +where+ and most of the other methods return a +Chain+
133
+ # the only way to get a result as a collection is to call the +all+
134
+ # method. It returns +Enumerator+ which could be used directly or
135
+ # transformed into +Array+
136
+ #
137
+ # Post.all # => Enumerator
138
+ # Post.where(links_count: 2).all # => Enumerator
139
+ # Post.where(links_count: 2).all.to_a # => Array
140
+ #
141
+ # When the result set is too large DynamoDB divides it into separate
142
+ # pages. While an enumerator iterates over the result models each page
143
+ # is loaded lazily. So even an extra large result set can be loaded and
144
+ # processed with considerably small memory footprint and throughput
145
+ # consumption.
146
+ #
147
+ # @return [Enumerator::Lazy]
77
148
  # @since 0.2.0
78
149
  def all
79
150
  records
80
151
  end
81
152
 
153
+ # Returns the actual number of items in a table matching the criteria.
154
+ #
155
+ # Post.where(links_count: 2).count
156
+ #
157
+ # Internally it uses either `Scan` or `Query` DynamoDB's operation so it
158
+ # costs like all the matching items were read from a table.
159
+ #
160
+ # The only difference is that items are read by DynemoDB but not actually
161
+ # loaded on the client side. DynamoDB returns only count of items after
162
+ # filtering.
163
+ #
164
+ # @return [Integer]
82
165
  def count
83
166
  if @key_fields_detector.key_present?
84
167
  count_via_query
@@ -87,27 +170,72 @@ module Dynamoid
87
170
  end
88
171
  end
89
172
 
90
- # Returns the last fetched record matched the criteria
91
- # Enumerable doesn't implement `last`, only `first`
92
- # So we have to implement it ourselves
173
+ # Returns the first item matching the criteria.
174
+ #
175
+ # Post.where(links_count: 2).first
176
+ #
177
+ # Applies `record_limit(1)` to ensure only a single record is fetched
178
+ # when no non-key conditions are present and `scan_limit(1)` when no
179
+ # conditions are present at all.
180
+ #
181
+ # If used without criteria it just returns the first item of some
182
+ # arbitrary order.
93
183
  #
184
+ # Post.first
185
+ #
186
+ # @return [Model|nil]
187
+ def first(*args)
188
+ n = args.first || 1
189
+
190
+ return scan_limit(n).to_a.first(*args) if @query.blank?
191
+ return super if @key_fields_detector.non_key_present?
192
+
193
+ record_limit(n).to_a.first(*args)
194
+ end
195
+
196
+ # Returns the last item matching the criteria.
197
+ #
198
+ # Post.where(links_count: 2).last
199
+ #
200
+ # DynamoDB doesn't support ordering by some arbitrary attribute except a
201
+ # sort key. So this method is mostly useful during development and
202
+ # testing.
203
+ #
204
+ # If used without criteria it just returns the last item of some arbitrary order.
205
+ #
206
+ # Post.last
207
+ #
208
+ # It isn't efficient from the performance point of view as far as it reads and
209
+ # loads all the filtered items from DynamoDB.
210
+ #
211
+ # @return [Model|nil]
94
212
  def last
95
213
  all.to_a.last
96
214
  end
97
215
 
98
- # Destroys all the records matching the criteria.
216
+ # Deletes all the items matching the criteria.
217
+ #
218
+ # Post.where(links_count: 2).delete_all
219
+ #
220
+ # If called without criteria then it deletes all the items in a table.
99
221
  #
222
+ # Post.delete_all
223
+ #
224
+ # It loads all the items either with +Scan+ or +Query+ operation and
225
+ # deletes them in batch with +BatchWriteItem+ operation. +BatchWriteItem+
226
+ # is limited by request size and items count so it's quite possible the
227
+ # deletion will require several +BatchWriteItem+ calls.
100
228
  def delete_all
101
229
  ids = []
102
230
  ranges = []
103
231
 
104
232
  if @key_fields_detector.key_present?
105
- Dynamoid.adapter.query(source.table_name, range_query).flat_map{ |i| i }.collect do |hash|
233
+ Dynamoid.adapter.query(source.table_name, range_query).flat_map { |i| i }.collect do |hash|
106
234
  ids << hash[source.hash_key.to_sym]
107
235
  ranges << hash[source.range_key.to_sym] if source.range_key
108
236
  end
109
237
  else
110
- Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).flat_map{ |i| i }.collect do |hash|
238
+ Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).flat_map { |i| i }.collect do |hash|
111
239
  ids << hash[source.hash_key.to_sym]
112
240
  ranges << hash[source.range_key.to_sym] if source.range_key
113
241
  end
@@ -117,53 +245,259 @@ module Dynamoid
117
245
  end
118
246
  alias destroy_all delete_all
119
247
 
120
- # The record limit is the limit of evaluated records returned by the
121
- # query or scan.
248
+ # Set the record limit.
249
+ #
250
+ # The record limit is the limit of evaluated items returned by the
251
+ # +Query+ or +Scan+. In other words it's how many items should be
252
+ # returned in response.
253
+ #
254
+ # Post.where(links_count: 2).record_limit(1000) # => 1000 models
255
+ # Post.record_limit(1000) # => 1000 models
256
+ #
257
+ # It could be very inefficient in terms of HTTP requests in pathological
258
+ # cases. DynamoDB doesn't support out of the box the limits for items
259
+ # count after filtering. So it's possible to make a lot of HTTP requests
260
+ # to find items matching criteria and skip not matching. It means that
261
+ # the cost (read capacity units) is unpredictable.
262
+ #
263
+ # Because of such issues with performance and cost it's mostly useful in
264
+ # development and testing.
265
+ #
266
+ # When called without criteria it works like +scan_limit+.
267
+ #
268
+ # @return [Dynamoid::Criteria::Chain]
122
269
  def record_limit(limit)
123
270
  @record_limit = limit
124
271
  self
125
272
  end
126
273
 
127
- # The scan limit which is the limit of records that DynamoDB will
128
- # internally query or scan. This is different from the record limit
129
- # as with filtering DynamoDB may look at N scanned records but return 0
130
- # records if none pass the filter.
274
+ # Set the scan limit.
275
+ #
276
+ # The scan limit is the limit of records that DynamoDB will internally
277
+ # read with +Query+ or +Scan+. It's different from the record limit as
278
+ # with filtering DynamoDB may look at N scanned items but return 0
279
+ # items if none passes the filter. So it can return less items than was
280
+ # specified with the limit.
281
+ #
282
+ # Post.where(links_count: 2).scan_limit(1000) # => 850 models
283
+ # Post.scan_limit(1000) # => 1000 models
284
+ #
285
+ # By contrast with +record_limit+ the cost (read capacity units) and
286
+ # performance is predictable.
287
+ #
288
+ # When called without criteria it works like +record_limit+.
289
+ #
290
+ # @return [Dynamoid::Criteria::Chain]
131
291
  def scan_limit(limit)
132
292
  @scan_limit = limit
133
293
  self
134
294
  end
135
295
 
296
+ # Set the batch size.
297
+ #
298
+ # The batch size is a number of items which will be lazily loaded one by one.
299
+ # When the batch size is set then items will be loaded batch by batch of
300
+ # the specified size instead of relying on the default paging mechanism
301
+ # of DynamoDB.
302
+ #
303
+ # Post.where(links_count: 2).batch(1000).all.each do |post|
304
+ # # process a post
305
+ # end
306
+ #
307
+ # It's useful to limit memory usage or throughput consumption
308
+ #
309
+ # @return [Dynamoid::Criteria::Chain]
136
310
  def batch(batch_size)
137
311
  @batch_size = batch_size
138
312
  self
139
313
  end
140
314
 
315
+ # Set the start item.
316
+ #
317
+ # When the start item is set the items will be loaded starting right
318
+ # after the specified item.
319
+ #
320
+ # Post.where(links_count: 2).start(post)
321
+ #
322
+ # It can be used to implement an own pagination mechanism.
323
+ #
324
+ # Post.where(author_id: author_id).start(last_post).scan_limit(50)
325
+ #
326
+ # The specified start item will not be returned back in a result set.
327
+ #
328
+ # Actually it doesn't need all the item attributes to start - an item may
329
+ # have only the primary key attributes (partition and sort key if it's
330
+ # declared).
331
+ #
332
+ # Post.where(links_count: 2).start(Post.new(id: id))
333
+ #
334
+ # It also supports a +Hash+ argument with the keys attributes - a
335
+ # partition key and a sort key (if it's declared).
336
+ #
337
+ # Post.where(links_count: 2).start(id: id)
338
+ #
339
+ # @return [Dynamoid::Criteria::Chain]
141
340
  def start(start)
142
341
  @start = start
143
342
  self
144
343
  end
145
344
 
345
+ # Reverse the sort order.
346
+ #
347
+ # By default the sort order is ascending (by the sort key value). Set a
348
+ # +false+ value to reverse the order.
349
+ #
350
+ # Post.where(id: id, 'views_count.gt' => 1000).scan_index_forward(false)
351
+ #
352
+ # It works only for queries with a partition key condition e.g. +id:
353
+ # 'some-id'+ which internally performs +Query+ operation.
354
+ #
355
+ # @return [Dynamoid::Criteria::Chain]
146
356
  def scan_index_forward(scan_index_forward)
147
357
  @scan_index_forward = scan_index_forward
148
358
  self
149
359
  end
150
360
 
151
- # Allows you to use the results of a search as an enumerable over the results found.
361
+ # Force the index name to use for queries.
362
+ #
363
+ # By default allows the library to select the most appropriate index.
364
+ # Sometimes you have more than one index which will fulfill your query's
365
+ # needs. When this case occurs you may want to force an order. This occurs
366
+ # when you are searching by hash key, but not specifying a range key.
367
+ #
368
+ # class Comment
369
+ # include Dynamoid::Document
370
+ #
371
+ # table key: :post_id
372
+ # range_key :author_id
373
+ #
374
+ # field :post_date, :datetime
375
+ #
376
+ # global_secondary_index name: :time_sorted_comments, hash_key: :post_id, range_key: post_date, projected_attributes: :all
377
+ # end
378
+ #
379
+ #
380
+ # Comment.where(post_id: id).with_index(:time_sorted_comments).scan_index_forward(false)
381
+ #
382
+ # @return [Dynamoid::Criteria::Chain]
383
+ def with_index(index_name)
384
+ raise Dynamoid::Errors::InvalidIndex, "Unknown index #{index_name}" unless @source.find_index_by_name(index_name)
385
+
386
+ @forced_index_name = index_name
387
+ @key_fields_detector = KeyFieldsDetector.new(@query, @source, forced_index_name: index_name)
388
+ self
389
+ end
390
+
391
+ # Allows to use the results of a search as an enumerable over the results
392
+ # found.
393
+ #
394
+ # Post.each do |post|
395
+ # end
396
+ #
397
+ # Post.all.each do |post|
398
+ # end
399
+ #
400
+ # Post.where(links_count: 2).each do |post|
401
+ # end
402
+ #
403
+ # It works similar to the +all+ method so results are loaded lazily.
152
404
  #
153
405
  # @since 0.2.0
154
406
  def each(&block)
155
407
  records.each(&block)
156
408
  end
157
409
 
410
+ # Iterates over the pages returned by DynamoDB.
411
+ #
412
+ # DynamoDB has its own paging machanism and divides a large result set
413
+ # into separate pages. The +find_by_pages+ method provides access to
414
+ # these native DynamoDB pages.
415
+ #
416
+ # The pages are loaded lazily.
417
+ #
418
+ # Post.where('views_count.gt' => 1000).find_by_pages do |posts, options|
419
+ # # process posts
420
+ # end
421
+ #
422
+ # It passes as block argument an +Array+ of models and a Hash with options.
423
+ #
424
+ # Options +Hash+ contains only one option +:last_evaluated_key+. The last
425
+ # evaluated key is a Hash with key attributes of the last item processed by
426
+ # DynamoDB. It can be used to resume querying using the +start+ method.
427
+ #
428
+ # posts, options = Post.where('views_count.gt' => 1000).find_by_pages.first
429
+ # last_key = options[:last_evaluated_key]
430
+ #
431
+ # # ...
432
+ #
433
+ # Post.where('views_count.gt' => 1000).start(last_key).find_by_pages do |posts, options|
434
+ # end
435
+ #
436
+ # If it's called without a block then it returns an +Enumerator+.
437
+ #
438
+ # enum = Post.where('views_count.gt' => 1000).find_by_pages
439
+ #
440
+ # enum.each do |posts, options|
441
+ # # process posts
442
+ # end
443
+ #
444
+ # @return [Enumerator::Lazy]
158
445
  def find_by_pages(&block)
159
446
  pages.each(&block)
160
447
  end
161
448
 
449
+ # Select only specified fields.
450
+ #
451
+ # It takes one or more field names and returns a collection of models with only
452
+ # these fields set.
453
+ #
454
+ # Post.where('views_count.gt' => 1000).select(:title)
455
+ # Post.where('views_count.gt' => 1000).select(:title, :created_at)
456
+ # Post.select(:id)
457
+ #
458
+ # It can be used to avoid loading large field values and to decrease a
459
+ # memory footprint.
460
+ #
461
+ # @return [Dynamoid::Criteria::Chain]
162
462
  def project(*fields)
163
463
  @project = fields.map(&:to_sym)
164
464
  self
165
465
  end
166
466
 
467
+ # Select only specified fields.
468
+ #
469
+ # It takes one or more field names and returns an array of either values
470
+ # or arrays of values.
471
+ #
472
+ # Post.pluck(:id) # => ['1', '2']
473
+ # Post.pluck(:title, :title) # => [['1', 'Title #1'], ['2', 'Title#2']]
474
+ #
475
+ # Post.where('views_count.gt' => 1000).pluck(:title)
476
+ #
477
+ # There are some differences between +pluck+ and +project+. +pluck+
478
+ # - doesn't instantiate models
479
+ # - it isn't chainable and returns +Array+ instead of +Chain+
480
+ #
481
+ # It deserializes values if a field type isn't supported by DynamoDB natively.
482
+ #
483
+ # It can be used to avoid loading large field values and to decrease a
484
+ # memory footprint.
485
+ #
486
+ # @return [Array]
487
+ def pluck(*args)
488
+ fields = args.map(&:to_sym)
489
+ @project = fields
490
+
491
+ if fields.many?
492
+ items.map do |item|
493
+ fields.map { |key| Undumping.undump_field(item[key], source.attributes[key]) }
494
+ end.to_a
495
+ else
496
+ key = fields.first
497
+ items.map { |item| Undumping.undump_field(item[key], source.attributes[key]) }.to_a
498
+ end
499
+ end
500
+
167
501
  private
168
502
 
169
503
  # The actual records referenced by the association.
@@ -172,7 +506,12 @@ module Dynamoid
172
506
  #
173
507
  # @since 0.2.0
174
508
  def records
175
- pages.lazy.flat_map { |i| i }
509
+ pages.lazy.flat_map { |items, _| items }
510
+ end
511
+
512
+ # Raw items like they are stored before type casting
513
+ def items
514
+ raw_pages.lazy.flat_map { |items, _| items }
176
515
  end
177
516
 
178
517
  # Arrays of records, sized based on the actual pages produced by DynamoDB
@@ -181,11 +520,19 @@ module Dynamoid
181
520
  #
182
521
  # @since 3.1.0
183
522
  def pages
523
+ raw_pages.lazy.map do |items, options|
524
+ models = items.map { |i| source.from_database(i) }
525
+ [models, options]
526
+ end.each
527
+ end
528
+
529
+ # Pages of items before type casting
530
+ def raw_pages
184
531
  if @key_fields_detector.key_present?
185
- pages_via_query
532
+ raw_pages_via_query
186
533
  else
187
534
  issue_scan_warning if Dynamoid::Config.warn_on_scan && query.present?
188
- pages_via_scan
535
+ raw_pages_via_scan
189
536
  end
190
537
  end
191
538
 
@@ -194,13 +541,12 @@ module Dynamoid
194
541
  # @return [Enumerator] an iterator of the found pages. An array of records
195
542
  #
196
543
  # @since 3.1.0
197
- def pages_via_query
544
+ def raw_pages_via_query
198
545
  Enumerator.new do |y|
199
546
  Dynamoid.adapter.query(source.table_name, range_query).each do |items, metadata|
200
- page = items.map { |h| source.from_database(h) }
201
547
  options = metadata.slice(:last_evaluated_key)
202
548
 
203
- y.yield page, options
549
+ y.yield items, options
204
550
  end
205
551
  end
206
552
  end
@@ -210,13 +556,12 @@ module Dynamoid
210
556
  # @return [Enumerator] an iterator of the found pages. An array of records
211
557
  #
212
558
  # @since 3.1.0
213
- def pages_via_scan
559
+ def raw_pages_via_scan
214
560
  Enumerator.new do |y|
215
561
  Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).each do |items, metadata|
216
- page = items.map { |h| source.from_database(h) }
217
562
  options = metadata.slice(:last_evaluated_key)
218
563
 
219
- y.yield page, options
564
+ y.yield items, options
220
565
  end
221
566
  end
222
567
  end
@@ -300,6 +645,13 @@ module Dynamoid
300
645
 
301
646
  def range_query
302
647
  opts = {}
648
+ query = self.query
649
+
650
+ # Honor STI and :type field if it presents
651
+ if @source.attributes.key?(@source.inheritance_field) &&
652
+ @key_fields_detector.hash_key.to_sym != @source.inheritance_field.to_sym
653
+ query.update(sti_condition)
654
+ end
303
655
 
304
656
  # Add hash key
305
657
  opts[:hash_key] = @key_fields_detector.hash_key
@@ -307,15 +659,7 @@ module Dynamoid
307
659
 
308
660
  # Add range key
309
661
  if @key_fields_detector.range_key
310
- opts[:range_key] = @key_fields_detector.range_key
311
- if query[@key_fields_detector.range_key].present?
312
- value = type_cast_condition_parameter(@key_fields_detector.range_key, query[@key_fields_detector.range_key])
313
- opts.update(range_eq: value)
314
- end
315
-
316
- query.keys.select { |k| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }.each do |key|
317
- opts.merge!(range_hash(key))
318
- end
662
+ add_range_key_to_range_query(query, opts)
319
663
  end
320
664
 
321
665
  (query.keys.map(&:to_sym) - [@key_fields_detector.hash_key.to_sym, @key_fields_detector.range_key.try(:to_sym)])
@@ -332,7 +676,19 @@ module Dynamoid
332
676
  opts.merge(query_opts).merge(consistent_opts)
333
677
  end
334
678
 
335
- # TODO casting should be operator aware
679
+ def add_range_key_to_range_query(query, opts)
680
+ opts[:range_key] = @key_fields_detector.range_key
681
+ if query[@key_fields_detector.range_key].present?
682
+ value = type_cast_condition_parameter(@key_fields_detector.range_key, query[@key_fields_detector.range_key])
683
+ opts.update(range_eq: value)
684
+ end
685
+
686
+ query.keys.select { |k| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }.each do |key|
687
+ opts.merge!(range_hash(key))
688
+ end
689
+ end
690
+
691
+ # TODO: casting should be operator aware
336
692
  # e.g. for NULL operator value should be boolean
337
693
  # and isn't related to an attribute own type
338
694
  def type_cast_condition_parameter(key, value)
@@ -393,6 +749,13 @@ module Dynamoid
393
749
  end
394
750
 
395
751
  def scan_query
752
+ query = self.query
753
+
754
+ # Honor STI and :type field if it presents
755
+ if sti_condition
756
+ query.update(sti_condition)
757
+ end
758
+
396
759
  {}.tap do |opts|
397
760
  query.keys.map(&:to_sym).each do |key|
398
761
  if key.to_s.include?('.')
@@ -415,6 +778,18 @@ module Dynamoid
415
778
  opts[:project] = @project
416
779
  opts
417
780
  end
781
+
782
+ def sti_condition
783
+ condition = {}
784
+ type = @source.inheritance_field
785
+
786
+ if @source.attributes.key?(type)
787
+ class_names = @source.deep_subclasses.map(&:name) << @source.name
788
+ condition[:"#{type}.in"] = class_names
789
+ end
790
+
791
+ condition
792
+ end
418
793
  end
419
794
  end
420
795
  end