dynamoid 3.8.0 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +89 -2
- data/README.md +375 -64
- data/SECURITY.md +17 -0
- data/dynamoid.gemspec +65 -0
- data/lib/dynamoid/adapter.rb +21 -14
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/create_table.rb +2 -2
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/execute_statement.rb +62 -0
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/filter_expression_convertor.rb +113 -0
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/item_updater.rb +29 -2
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/middleware/limit.rb +3 -0
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/projection_expression_convertor.rb +40 -0
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb +46 -61
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb +34 -28
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3/transact.rb +31 -0
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb +132 -74
- data/lib/dynamoid/associations/belongs_to.rb +6 -6
- data/lib/dynamoid/associations.rb +1 -1
- data/lib/dynamoid/components.rb +3 -3
- data/lib/dynamoid/config/options.rb +12 -12
- data/lib/dynamoid/config.rb +4 -0
- data/lib/dynamoid/criteria/chain.rb +165 -149
- data/lib/dynamoid/criteria/key_fields_detector.rb +6 -7
- data/lib/dynamoid/criteria/nonexistent_fields_detector.rb +2 -2
- data/lib/dynamoid/criteria/where_conditions.rb +36 -0
- data/lib/dynamoid/dirty.rb +145 -59
- data/lib/dynamoid/document.rb +39 -3
- data/lib/dynamoid/dumping.rb +41 -19
- data/lib/dynamoid/errors.rb +32 -3
- data/lib/dynamoid/fields/declare.rb +6 -6
- data/lib/dynamoid/fields.rb +21 -29
- data/lib/dynamoid/finders.rb +68 -51
- data/lib/dynamoid/indexes.rb +7 -10
- data/lib/dynamoid/loadable.rb +3 -2
- data/lib/dynamoid/log/formatter.rb +19 -4
- data/lib/dynamoid/persistence/import.rb +4 -1
- data/lib/dynamoid/persistence/inc.rb +82 -0
- data/lib/dynamoid/persistence/item_updater_with_casting_and_dumping.rb +36 -0
- data/lib/dynamoid/persistence/item_updater_with_dumping.rb +33 -0
- data/lib/dynamoid/persistence/save.rb +75 -17
- data/lib/dynamoid/persistence/update_fields.rb +24 -9
- data/lib/dynamoid/persistence/update_validations.rb +3 -3
- data/lib/dynamoid/persistence/upsert.rb +22 -8
- data/lib/dynamoid/persistence.rb +308 -72
- data/lib/dynamoid/transaction_read/find.rb +137 -0
- data/lib/dynamoid/transaction_read.rb +146 -0
- data/lib/dynamoid/transaction_write/base.rb +47 -0
- data/lib/dynamoid/transaction_write/create.rb +49 -0
- data/lib/dynamoid/transaction_write/delete_with_instance.rb +65 -0
- data/lib/dynamoid/transaction_write/delete_with_primary_key.rb +64 -0
- data/lib/dynamoid/transaction_write/destroy.rb +84 -0
- data/lib/dynamoid/transaction_write/item_updater.rb +55 -0
- data/lib/dynamoid/transaction_write/save.rb +169 -0
- data/lib/dynamoid/transaction_write/update_attributes.rb +46 -0
- data/lib/dynamoid/transaction_write/update_fields.rb +239 -0
- data/lib/dynamoid/transaction_write/upsert.rb +106 -0
- data/lib/dynamoid/transaction_write.rb +673 -0
- data/lib/dynamoid/type_casting.rb +18 -15
- data/lib/dynamoid/undumping.rb +14 -3
- data/lib/dynamoid/validations.rb +8 -5
- data/lib/dynamoid/version.rb +1 -1
- data/lib/dynamoid.rb +8 -0
- metadata +43 -49
- data/lib/dynamoid/criteria/ignored_conditions_detector.rb +0 -41
- data/lib/dynamoid/criteria/overwritten_conditions_detector.rb +0 -40
data/dynamoid.gemspec
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
|
+
require 'dynamoid/version'
|
|
6
|
+
|
|
7
|
+
Gem::Specification.new do |spec|
|
|
8
|
+
spec.name = 'dynamoid'
|
|
9
|
+
spec.version = Dynamoid::VERSION
|
|
10
|
+
|
|
11
|
+
# Keep in sync with README
|
|
12
|
+
spec.authors = [
|
|
13
|
+
'Josh Symonds',
|
|
14
|
+
'Logan Bowers',
|
|
15
|
+
'Craig Heneveld',
|
|
16
|
+
'Anatha Kumaran',
|
|
17
|
+
'Jason Dew',
|
|
18
|
+
'Luis Arias',
|
|
19
|
+
'Stefan Neculai',
|
|
20
|
+
'Philip White',
|
|
21
|
+
'Peeyush Kumar',
|
|
22
|
+
'Sumanth Ravipati',
|
|
23
|
+
'Pascal Corpet',
|
|
24
|
+
'Brian Glusman',
|
|
25
|
+
'Peter Boling',
|
|
26
|
+
'Andrew Konchin'
|
|
27
|
+
]
|
|
28
|
+
spec.email = ['andry.konchin@gmail.com', 'peter.boling@gmail.com', 'brian@stellaservice.com']
|
|
29
|
+
|
|
30
|
+
spec.description = "Dynamoid is an ORM for Amazon's DynamoDB that supports offline development, associations, querying, and everything else you'd expect from an ActiveRecord-style replacement."
|
|
31
|
+
spec.summary = "Dynamoid is an ORM for Amazon's DynamoDB"
|
|
32
|
+
# Ignore not committed files
|
|
33
|
+
spec.files = Dir[
|
|
34
|
+
'CHANGELOG.md',
|
|
35
|
+
'dynamoid.gemspec',
|
|
36
|
+
'lib/**/*',
|
|
37
|
+
'LICENSE.txt',
|
|
38
|
+
'README.md',
|
|
39
|
+
'SECURITY.md'
|
|
40
|
+
]
|
|
41
|
+
spec.homepage = 'http://github.com/Dynamoid/dynamoid'
|
|
42
|
+
spec.licenses = ['MIT']
|
|
43
|
+
spec.require_paths = ['lib']
|
|
44
|
+
|
|
45
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
|
46
|
+
spec.metadata['source_code_uri'] = "https://github.com/Dynamoid/dynamoid/tree/v#{spec.version}"
|
|
47
|
+
spec.metadata['changelog_uri'] = "https://github.com/Dynamoid/dynamoid/blob/v#{spec.version}/CHANGELOG.md"
|
|
48
|
+
spec.metadata['bug_tracker_uri'] = 'https://github.com/Dynamoid/dynamoid/issues'
|
|
49
|
+
spec.metadata['documentation_uri'] = "https://www.rubydoc.info/gems/dynamoid/#{spec.version}"
|
|
50
|
+
spec.metadata['funding_uri'] = 'https://opencollective.com/dynamoid'
|
|
51
|
+
spec.metadata['wiki_uri'] = 'https://github.com/Dynamoid/dynamoid/wiki'
|
|
52
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
|
53
|
+
|
|
54
|
+
spec.add_dependency 'activemodel', '>=4'
|
|
55
|
+
spec.add_dependency 'aws-sdk-dynamodb', '~> 1.0'
|
|
56
|
+
spec.add_dependency 'concurrent-ruby', '>= 1.0'
|
|
57
|
+
|
|
58
|
+
spec.add_development_dependency 'appraisal'
|
|
59
|
+
spec.add_development_dependency 'bundler'
|
|
60
|
+
spec.add_development_dependency 'pry', '~> 0.14'
|
|
61
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
|
62
|
+
spec.add_development_dependency 'rexml'
|
|
63
|
+
spec.add_development_dependency 'rspec', '~> 3.12'
|
|
64
|
+
spec.add_development_dependency 'yard'
|
|
65
|
+
end
|
data/lib/dynamoid/adapter.rb
CHANGED
|
@@ -118,7 +118,7 @@ module Dynamoid
|
|
|
118
118
|
# @param [Hash] query a hash of attributes: matching records will be returned by the scan
|
|
119
119
|
#
|
|
120
120
|
# @since 0.2.0
|
|
121
|
-
def scan(table, query =
|
|
121
|
+
def scan(table, query = [], opts = {})
|
|
122
122
|
benchmark('Scan', table, query) { adapter.scan(table, query, opts) }
|
|
123
123
|
end
|
|
124
124
|
|
|
@@ -142,7 +142,7 @@ module Dynamoid
|
|
|
142
142
|
end
|
|
143
143
|
end
|
|
144
144
|
|
|
145
|
-
%i[batch_get_item delete_item get_item list_tables put_item truncate batch_write_item batch_delete_item].each do |m|
|
|
145
|
+
%i[batch_get_item delete_item get_item list_tables put_item truncate batch_write_item batch_delete_item execute].each do |m|
|
|
146
146
|
# Method delegation with benchmark to the underlying adapter. Faster than relying on method_missing.
|
|
147
147
|
#
|
|
148
148
|
# @since 0.2.0
|
|
@@ -163,6 +163,7 @@ module Dynamoid
|
|
|
163
163
|
# https://eregon.me/blog/2019/11/10/the-delegation-challenge-of-ruby27.html
|
|
164
164
|
|
|
165
165
|
return benchmark(method, *args) { adapter.send(method, *args, &block) } if adapter.respond_to?(method)
|
|
166
|
+
|
|
166
167
|
super
|
|
167
168
|
end
|
|
168
169
|
|
|
@@ -170,19 +171,25 @@ module Dynamoid
|
|
|
170
171
|
# only really useful for range queries, since it can only find by one hash key at once. Only provide
|
|
171
172
|
# one range key to the hash.
|
|
172
173
|
#
|
|
174
|
+
# Dynamoid.adapter.query('users', { id: [[:eq, '1']], age: [[:between, [10, 30]]] }, { batch_size: 1000 })
|
|
175
|
+
#
|
|
173
176
|
# @param [String] table_name the name of the table
|
|
174
|
-
# @param [
|
|
175
|
-
# @
|
|
176
|
-
# @
|
|
177
|
-
# @option
|
|
178
|
-
# @option
|
|
179
|
-
# @option
|
|
180
|
-
# @option
|
|
181
|
-
#
|
|
182
|
-
# @
|
|
183
|
-
#
|
|
184
|
-
|
|
185
|
-
|
|
177
|
+
# @param [Array[Array]] key_conditions conditions for the primary key attributes
|
|
178
|
+
# @param [Array[Array]] non_key_conditions (optional) conditions for non-primary key attributes
|
|
179
|
+
# @param [Hash] options (optional) the options to query the table with
|
|
180
|
+
# @option options [Boolean] :consistent_read You can set the ConsistentRead parameter to true and obtain a strongly consistent result
|
|
181
|
+
# @option options [Boolean] :scan_index_forward Specifies the order for index traversal: If true (default), the traversal is performed in ascending order; if false, the traversal is performed in descending order.
|
|
182
|
+
# @option options [Symbop] :select The attributes to be returned in the result (one of ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES, ...)
|
|
183
|
+
# @option options [Symbol] :index_name The name of an index to query. This index can be any local secondary index or global secondary index on the table.
|
|
184
|
+
# @option options [Hash] :exclusive_start_key The primary key of the first item that this operation will evaluate.
|
|
185
|
+
# @option options [Integer] :batch_size The number of items to lazily load one by one
|
|
186
|
+
# @option options [Integer] :record_limit The maximum number of items to return (not necessarily the number of evaluated items)
|
|
187
|
+
# @option options [Integer] :scan_limit The maximum number of items to evaluate (not necessarily the number of matching items)
|
|
188
|
+
# @option options [Array[Symbol]] :project The attributes to retrieve from the table
|
|
189
|
+
#
|
|
190
|
+
# @return [Enumerable] matching items
|
|
191
|
+
def query(table_name, key_conditions, non_key_conditions = {}, options = {})
|
|
192
|
+
adapter.query(table_name, key_conditions, non_key_conditions, options)
|
|
186
193
|
end
|
|
187
194
|
|
|
188
195
|
def self.adapter_plugin_class
|
|
@@ -29,7 +29,7 @@ module Dynamoid
|
|
|
29
29
|
gs_indexes = options[:global_secondary_indexes]
|
|
30
30
|
|
|
31
31
|
key_schema = {
|
|
32
|
-
hash_key_schema: { key =>
|
|
32
|
+
hash_key_schema: { key => options[:hash_key_type] || :string },
|
|
33
33
|
range_key_schema: options[:range_key]
|
|
34
34
|
}
|
|
35
35
|
attribute_definitions = build_all_attribute_definitions(
|
|
@@ -69,7 +69,7 @@ module Dynamoid
|
|
|
69
69
|
end
|
|
70
70
|
end
|
|
71
71
|
resp = client.create_table(client_opts)
|
|
72
|
-
options[:sync] = true if !options.key?(:sync) && ls_indexes.present? || gs_indexes.present?
|
|
72
|
+
options[:sync] = true if (!options.key?(:sync) && ls_indexes.present?) || gs_indexes.present?
|
|
73
73
|
|
|
74
74
|
if options[:sync]
|
|
75
75
|
status = PARSE_TABLE_STATUS.call(resp, :table_description)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Dynamoid
|
|
4
|
+
# @private
|
|
5
|
+
module AdapterPlugin
|
|
6
|
+
class AwsSdkV3
|
|
7
|
+
# Excecute a PartiQL query
|
|
8
|
+
#
|
|
9
|
+
# Documentation:
|
|
10
|
+
# - https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_ExecuteStatement.html
|
|
11
|
+
# - https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#execute_statement-instance_method
|
|
12
|
+
#
|
|
13
|
+
# NOTE: For reads result may be paginated. Only pagination with NextToken
|
|
14
|
+
# is implemented. Currently LastEvaluatedKey in response cannot be fed to
|
|
15
|
+
# ExecuteStatement to get the next page.
|
|
16
|
+
#
|
|
17
|
+
# See also:
|
|
18
|
+
# - https://repost.aws/questions/QUgNPbBYWiRoOlMsJv-XzrWg/how-to-use-last-evaluated-key-in-execute-statement-request
|
|
19
|
+
# - https://stackoverflow.com/questions/71438439/aws-dynamodb-executestatement-pagination
|
|
20
|
+
class ExecuteStatement
|
|
21
|
+
attr_reader :client, :statement, :parameters, :options
|
|
22
|
+
|
|
23
|
+
def initialize(client, statement, parameters, options)
|
|
24
|
+
@client = client
|
|
25
|
+
@statement = statement
|
|
26
|
+
@parameters = parameters
|
|
27
|
+
@options = options.symbolize_keys.slice(:consistent_read)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def call
|
|
31
|
+
request = {
|
|
32
|
+
statement: @statement,
|
|
33
|
+
parameters: @parameters,
|
|
34
|
+
consistent_read: @options[:consistent_read],
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
response = client.execute_statement(request)
|
|
38
|
+
|
|
39
|
+
unless response.next_token
|
|
40
|
+
return response_to_items(response)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
Enumerator.new do |yielder|
|
|
44
|
+
yielder.yield(response_to_items(response))
|
|
45
|
+
|
|
46
|
+
while response.next_token
|
|
47
|
+
request[:next_token] = response.next_token
|
|
48
|
+
response = client.execute_statement(request)
|
|
49
|
+
yielder.yield(response_to_items(response))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def response_to_items(response)
|
|
57
|
+
response.items.map(&:symbolize_keys)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Dynamoid
|
|
4
|
+
# @private
|
|
5
|
+
module AdapterPlugin
|
|
6
|
+
class AwsSdkV3
|
|
7
|
+
class FilterExpressionConvertor
|
|
8
|
+
attr_reader :expression, :name_placeholders, :value_placeholders
|
|
9
|
+
|
|
10
|
+
def initialize(conditions, name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence)
|
|
11
|
+
@conditions = conditions
|
|
12
|
+
@name_placeholders = name_placeholders.dup
|
|
13
|
+
@value_placeholders = value_placeholders.dup
|
|
14
|
+
@name_placeholder_sequence = name_placeholder_sequence
|
|
15
|
+
@value_placeholder_sequence = value_placeholder_sequence
|
|
16
|
+
|
|
17
|
+
build
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def build
|
|
23
|
+
clauses = []
|
|
24
|
+
|
|
25
|
+
@conditions.each do |conditions|
|
|
26
|
+
if conditions.is_a? Hash
|
|
27
|
+
clauses << build_for_hash(conditions) unless conditions.empty?
|
|
28
|
+
elsif conditions.is_a? Array
|
|
29
|
+
query, placeholders = conditions
|
|
30
|
+
clauses << build_for_string(query, placeholders)
|
|
31
|
+
else
|
|
32
|
+
raise ArgumentError, "expected Hash or Array but actual value is #{conditions}"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
@expression = clauses.join(' AND ')
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def build_for_hash(hash)
|
|
40
|
+
clauses = hash.map do |name, attribute_conditions|
|
|
41
|
+
attribute_conditions.map do |operator, value|
|
|
42
|
+
# replace attribute names with placeholders unconditionally to support
|
|
43
|
+
# - special characters (e.g. '.', ':', and '#') and
|
|
44
|
+
# - leading '_'
|
|
45
|
+
# See
|
|
46
|
+
# - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.NamingRules
|
|
47
|
+
# - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html#Expressions.ExpressionAttributeNames.AttributeNamesContainingSpecialCharacters
|
|
48
|
+
name_placeholder = name_placeholder_for(name)
|
|
49
|
+
|
|
50
|
+
case operator
|
|
51
|
+
when :eq
|
|
52
|
+
"#{name_placeholder} = #{value_placeholder_for(value)}"
|
|
53
|
+
when :ne
|
|
54
|
+
"#{name_placeholder} <> #{value_placeholder_for(value)}"
|
|
55
|
+
when :gt
|
|
56
|
+
"#{name_placeholder} > #{value_placeholder_for(value)}"
|
|
57
|
+
when :lt
|
|
58
|
+
"#{name_placeholder} < #{value_placeholder_for(value)}"
|
|
59
|
+
when :gte
|
|
60
|
+
"#{name_placeholder} >= #{value_placeholder_for(value)}"
|
|
61
|
+
when :lte
|
|
62
|
+
"#{name_placeholder} <= #{value_placeholder_for(value)}"
|
|
63
|
+
when :between
|
|
64
|
+
"#{name_placeholder} BETWEEN #{value_placeholder_for(value[0])} AND #{value_placeholder_for(value[1])}"
|
|
65
|
+
when :begins_with
|
|
66
|
+
"begins_with (#{name_placeholder}, #{value_placeholder_for(value)})"
|
|
67
|
+
when :in
|
|
68
|
+
list = value.map(&method(:value_placeholder_for)).join(' , ')
|
|
69
|
+
"#{name_placeholder} IN (#{list})"
|
|
70
|
+
when :contains
|
|
71
|
+
"contains (#{name_placeholder}, #{value_placeholder_for(value)})"
|
|
72
|
+
when :not_contains
|
|
73
|
+
"NOT contains (#{name_placeholder}, #{value_placeholder_for(value)})"
|
|
74
|
+
when :null
|
|
75
|
+
"attribute_not_exists (#{name_placeholder})"
|
|
76
|
+
when :not_null
|
|
77
|
+
"attribute_exists (#{name_placeholder})"
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end.flatten
|
|
81
|
+
|
|
82
|
+
if clauses.empty?
|
|
83
|
+
nil
|
|
84
|
+
else
|
|
85
|
+
clauses.join(' AND ')
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def build_for_string(query, placeholders)
|
|
90
|
+
placeholders.each do |(k, v)|
|
|
91
|
+
k = k.to_s
|
|
92
|
+
k = ":#{k}" unless k.start_with?(':')
|
|
93
|
+
@value_placeholders[k] = v
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
"(#{query})"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def name_placeholder_for(name)
|
|
100
|
+
placeholder = @name_placeholder_sequence.call
|
|
101
|
+
@name_placeholders[placeholder] = name
|
|
102
|
+
placeholder
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def value_placeholder_for(value)
|
|
106
|
+
placeholder = @value_placeholder_sequence.call
|
|
107
|
+
@value_placeholders[placeholder] = value
|
|
108
|
+
placeholder
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -50,7 +50,19 @@ module Dynamoid
|
|
|
50
50
|
# Replaces the values of one or more attributes
|
|
51
51
|
#
|
|
52
52
|
def set(values)
|
|
53
|
-
|
|
53
|
+
values_sanitized = sanitize_attributes(values)
|
|
54
|
+
|
|
55
|
+
if Dynamoid.config.store_attribute_with_nil_value
|
|
56
|
+
@updates.merge!(values_sanitized)
|
|
57
|
+
else
|
|
58
|
+
# delete explicitly attributes if assigned nil value and configured
|
|
59
|
+
# to not store nil values
|
|
60
|
+
values_to_update = values_sanitized.reject { |_, v| v.nil? }
|
|
61
|
+
values_to_delete = values_sanitized.select { |_, v| v.nil? }
|
|
62
|
+
|
|
63
|
+
@updates.merge!(values_to_update)
|
|
64
|
+
@deletions.merge!(values_to_delete)
|
|
65
|
+
end
|
|
54
66
|
end
|
|
55
67
|
|
|
56
68
|
#
|
|
@@ -85,10 +97,25 @@ module Dynamoid
|
|
|
85
97
|
|
|
86
98
|
private
|
|
87
99
|
|
|
100
|
+
# It's a single low level component available in a public API (with
|
|
101
|
+
# Document#update/#update! methods). So duplicate sanitizing to some
|
|
102
|
+
# degree.
|
|
103
|
+
#
|
|
104
|
+
# Keep in sync with AwsSdkV3.sanitize_item.
|
|
88
105
|
def sanitize_attributes(attributes)
|
|
106
|
+
# rubocop:disable Lint/DuplicateBranch
|
|
89
107
|
attributes.transform_values do |v|
|
|
90
|
-
v.is_a?(Hash)
|
|
108
|
+
if v.is_a?(Hash)
|
|
109
|
+
v.stringify_keys
|
|
110
|
+
elsif v.is_a?(Set) && v.empty?
|
|
111
|
+
nil
|
|
112
|
+
elsif v.is_a?(String) && v.empty? && Config.store_empty_string_as_nil
|
|
113
|
+
nil
|
|
114
|
+
else
|
|
115
|
+
v
|
|
116
|
+
end
|
|
91
117
|
end
|
|
118
|
+
# rubocop:enable Lint/DuplicateBranch
|
|
92
119
|
end
|
|
93
120
|
end
|
|
94
121
|
end
|
|
@@ -21,14 +21,17 @@ module Dynamoid
|
|
|
21
21
|
# lower to obey limits. We can assume the difference won't be
|
|
22
22
|
# negative due to break statements below but choose smaller limit
|
|
23
23
|
# which is why we have 2 separate if statements.
|
|
24
|
+
#
|
|
24
25
|
# NOTE: Adjusting based on record_limit can cause many HTTP requests
|
|
25
26
|
# being made. We may want to change this behavior, but it affects
|
|
26
27
|
# filtering on data with potentially large gaps.
|
|
28
|
+
#
|
|
27
29
|
# Example:
|
|
28
30
|
# User.where('created_at.gte' => 1.day.ago).record_limit(1000)
|
|
29
31
|
# Records 1-999 User's that fit criteria
|
|
30
32
|
# Records 1000-2000 Users's that do not fit criteria
|
|
31
33
|
# Record 2001 fits criteria
|
|
34
|
+
#
|
|
32
35
|
# The underlying implementation will have 1 page for records 1-999
|
|
33
36
|
# then will request with limit 1 for records 1000-2000 (making 1000
|
|
34
37
|
# requests of limit 1) until hit record 2001.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Dynamoid
|
|
4
|
+
# @private
|
|
5
|
+
module AdapterPlugin
|
|
6
|
+
class AwsSdkV3
|
|
7
|
+
class ProjectionExpressionConvertor
|
|
8
|
+
attr_reader :expression, :name_placeholders
|
|
9
|
+
|
|
10
|
+
def initialize(names, name_placeholders, name_placeholder_sequence)
|
|
11
|
+
@names = names
|
|
12
|
+
@name_placeholders = name_placeholders.dup
|
|
13
|
+
@name_placeholder_sequence = name_placeholder_sequence
|
|
14
|
+
|
|
15
|
+
build
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
private
|
|
19
|
+
|
|
20
|
+
def build
|
|
21
|
+
return if @names.nil? || @names.empty?
|
|
22
|
+
|
|
23
|
+
clauses = @names.map do |name|
|
|
24
|
+
# replace attribute names with placeholders unconditionally to support
|
|
25
|
+
# - special characters (e.g. '.', ':', and '#') and
|
|
26
|
+
# - leading '_'
|
|
27
|
+
# See
|
|
28
|
+
# - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.NamingRules
|
|
29
|
+
# - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html#Expressions.ExpressionAttributeNames.AttributeNamesContainingSpecialCharacters
|
|
30
|
+
placeholder = @name_placeholder_sequence.call
|
|
31
|
+
@name_placeholders[placeholder] = name
|
|
32
|
+
placeholder
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
@expression = clauses.join(' , ')
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require_relative 'middleware/backoff'
|
|
4
4
|
require_relative 'middleware/limit'
|
|
5
5
|
require_relative 'middleware/start_key'
|
|
6
|
+
require_relative 'filter_expression_convertor'
|
|
7
|
+
require_relative 'projection_expression_convertor'
|
|
6
8
|
|
|
7
9
|
module Dynamoid
|
|
8
10
|
# @private
|
|
@@ -10,20 +12,19 @@ module Dynamoid
|
|
|
10
12
|
class AwsSdkV3
|
|
11
13
|
class Query
|
|
12
14
|
OPTIONS_KEYS = %i[
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
project
|
|
15
|
+
consistent_read scan_index_forward select index_name batch_size
|
|
16
|
+
exclusive_start_key record_limit scan_limit project
|
|
16
17
|
].freeze
|
|
17
18
|
|
|
18
19
|
attr_reader :client, :table, :options, :conditions
|
|
19
20
|
|
|
20
|
-
def initialize(client, table,
|
|
21
|
+
def initialize(client, table, key_conditions, non_key_conditions, options)
|
|
21
22
|
@client = client
|
|
22
23
|
@table = table
|
|
23
24
|
|
|
24
|
-
|
|
25
|
-
@
|
|
26
|
-
@
|
|
25
|
+
@key_conditions = key_conditions
|
|
26
|
+
@non_key_conditions = non_key_conditions
|
|
27
|
+
@options = options.slice(*OPTIONS_KEYS)
|
|
27
28
|
end
|
|
28
29
|
|
|
29
30
|
def call
|
|
@@ -53,6 +54,37 @@ module Dynamoid
|
|
|
53
54
|
private
|
|
54
55
|
|
|
55
56
|
def build_request
|
|
57
|
+
# expressions
|
|
58
|
+
name_placeholder = +'#_a0'
|
|
59
|
+
value_placeholder = +':_a0'
|
|
60
|
+
|
|
61
|
+
name_placeholder_sequence = -> { name_placeholder.next!.dup }
|
|
62
|
+
value_placeholder_sequence = -> { value_placeholder.next!.dup }
|
|
63
|
+
|
|
64
|
+
name_placeholders = {}
|
|
65
|
+
value_placeholders = {}
|
|
66
|
+
|
|
67
|
+
# Deal with various limits and batching
|
|
68
|
+
batch_size = options[:batch_size]
|
|
69
|
+
limit = [record_limit, scan_limit, batch_size].compact.min
|
|
70
|
+
|
|
71
|
+
# key condition expression
|
|
72
|
+
convertor = FilterExpressionConvertor.new([@key_conditions], name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence)
|
|
73
|
+
key_condition_expression = convertor.expression
|
|
74
|
+
value_placeholders = convertor.value_placeholders
|
|
75
|
+
name_placeholders = convertor.name_placeholders
|
|
76
|
+
|
|
77
|
+
# filter expression
|
|
78
|
+
convertor = FilterExpressionConvertor.new(@non_key_conditions, name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence)
|
|
79
|
+
filter_expression = convertor.expression
|
|
80
|
+
value_placeholders = convertor.value_placeholders
|
|
81
|
+
name_placeholders = convertor.name_placeholders
|
|
82
|
+
|
|
83
|
+
# projection expression
|
|
84
|
+
convertor = ProjectionExpressionConvertor.new(options[:project], name_placeholders, name_placeholder_sequence)
|
|
85
|
+
projection_expression = convertor.expression
|
|
86
|
+
name_placeholders = convertor.name_placeholders
|
|
87
|
+
|
|
56
88
|
request = options.slice(
|
|
57
89
|
:consistent_read,
|
|
58
90
|
:scan_index_forward,
|
|
@@ -61,15 +93,13 @@ module Dynamoid
|
|
|
61
93
|
:exclusive_start_key
|
|
62
94
|
).compact
|
|
63
95
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
request[:
|
|
69
|
-
request[:
|
|
70
|
-
request[:
|
|
71
|
-
request[:query_filter] = query_filter
|
|
72
|
-
request[:attributes_to_get] = attributes_to_get
|
|
96
|
+
request[:table_name] = table.name
|
|
97
|
+
request[:limit] = limit if limit
|
|
98
|
+
request[:key_condition_expression] = key_condition_expression if key_condition_expression.present?
|
|
99
|
+
request[:filter_expression] = filter_expression if filter_expression.present?
|
|
100
|
+
request[:expression_attribute_values] = value_placeholders if value_placeholders.present?
|
|
101
|
+
request[:expression_attribute_names] = name_placeholders if name_placeholders.present?
|
|
102
|
+
request[:projection_expression] = projection_expression if projection_expression.present?
|
|
73
103
|
|
|
74
104
|
request
|
|
75
105
|
end
|
|
@@ -81,51 +111,6 @@ module Dynamoid
|
|
|
81
111
|
def scan_limit
|
|
82
112
|
options[:scan_limit]
|
|
83
113
|
end
|
|
84
|
-
|
|
85
|
-
def hash_key_name
|
|
86
|
-
(options[:hash_key] || table.hash_key)
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
def range_key_name
|
|
90
|
-
(options[:range_key] || table.range_key)
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
def key_conditions
|
|
94
|
-
result = {
|
|
95
|
-
hash_key_name => {
|
|
96
|
-
comparison_operator: AwsSdkV3::EQ,
|
|
97
|
-
attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::EQ, options[:hash_value].freeze)
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
conditions.slice(*AwsSdkV3::RANGE_MAP.keys).each do |k, _v|
|
|
102
|
-
op = AwsSdkV3::RANGE_MAP[k]
|
|
103
|
-
|
|
104
|
-
result[range_key_name] = {
|
|
105
|
-
comparison_operator: op,
|
|
106
|
-
attribute_value_list: AwsSdkV3.attribute_value_list(op, conditions[k].freeze)
|
|
107
|
-
}
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
result
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
def query_filter
|
|
114
|
-
conditions.except(*AwsSdkV3::RANGE_MAP.keys).reduce({}) do |result, (attr, cond)|
|
|
115
|
-
condition = {
|
|
116
|
-
comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
|
|
117
|
-
attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
|
|
118
|
-
}
|
|
119
|
-
result[attr] = condition
|
|
120
|
-
result
|
|
121
|
-
end
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
def attributes_to_get
|
|
125
|
-
return if options[:project].nil?
|
|
126
|
-
|
|
127
|
-
options[:project].map(&:to_s)
|
|
128
|
-
end
|
|
129
114
|
end
|
|
130
115
|
end
|
|
131
116
|
end
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require_relative 'middleware/backoff'
|
|
4
4
|
require_relative 'middleware/limit'
|
|
5
5
|
require_relative 'middleware/start_key'
|
|
6
|
+
require_relative 'filter_expression_convertor'
|
|
7
|
+
require_relative 'projection_expression_convertor'
|
|
6
8
|
|
|
7
9
|
module Dynamoid
|
|
8
10
|
# @private
|
|
@@ -11,7 +13,7 @@ module Dynamoid
|
|
|
11
13
|
class Scan
|
|
12
14
|
attr_reader :client, :table, :conditions, :options
|
|
13
15
|
|
|
14
|
-
def initialize(client, table, conditions =
|
|
16
|
+
def initialize(client, table, conditions = [], options = {})
|
|
15
17
|
@client = client
|
|
16
18
|
@table = table
|
|
17
19
|
@conditions = conditions
|
|
@@ -45,6 +47,31 @@ module Dynamoid
|
|
|
45
47
|
private
|
|
46
48
|
|
|
47
49
|
def build_request
|
|
50
|
+
# expressions
|
|
51
|
+
name_placeholder = +'#_a0'
|
|
52
|
+
value_placeholder = +':_a0'
|
|
53
|
+
|
|
54
|
+
name_placeholder_sequence = -> { name_placeholder.next!.dup }
|
|
55
|
+
value_placeholder_sequence = -> { value_placeholder.next!.dup }
|
|
56
|
+
|
|
57
|
+
name_placeholders = {}
|
|
58
|
+
value_placeholders = {}
|
|
59
|
+
|
|
60
|
+
# Deal with various limits and batching
|
|
61
|
+
batch_size = options[:batch_size]
|
|
62
|
+
limit = [record_limit, scan_limit, batch_size].compact.min
|
|
63
|
+
|
|
64
|
+
# filter expression
|
|
65
|
+
convertor = FilterExpressionConvertor.new(conditions, name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence)
|
|
66
|
+
filter_expression = convertor.expression
|
|
67
|
+
value_placeholders = convertor.value_placeholders
|
|
68
|
+
name_placeholders = convertor.name_placeholders
|
|
69
|
+
|
|
70
|
+
# projection expression
|
|
71
|
+
convertor = ProjectionExpressionConvertor.new(options[:project], name_placeholders, name_placeholder_sequence)
|
|
72
|
+
projection_expression = convertor.expression
|
|
73
|
+
name_placeholders = convertor.name_placeholders
|
|
74
|
+
|
|
48
75
|
request = options.slice(
|
|
49
76
|
:consistent_read,
|
|
50
77
|
:exclusive_start_key,
|
|
@@ -52,14 +79,12 @@ module Dynamoid
|
|
|
52
79
|
:index_name
|
|
53
80
|
).compact
|
|
54
81
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
request[:
|
|
60
|
-
request[:
|
|
61
|
-
request[:scan_filter] = scan_filter
|
|
62
|
-
request[:attributes_to_get] = attributes_to_get
|
|
82
|
+
request[:table_name] = table.name
|
|
83
|
+
request[:limit] = limit if limit
|
|
84
|
+
request[:filter_expression] = filter_expression if filter_expression.present?
|
|
85
|
+
request[:expression_attribute_values] = value_placeholders if value_placeholders.present?
|
|
86
|
+
request[:expression_attribute_names] = name_placeholders if name_placeholders.present?
|
|
87
|
+
request[:projection_expression] = projection_expression if projection_expression.present?
|
|
63
88
|
|
|
64
89
|
request
|
|
65
90
|
end
|
|
@@ -71,25 +96,6 @@ module Dynamoid
|
|
|
71
96
|
def scan_limit
|
|
72
97
|
options[:scan_limit]
|
|
73
98
|
end
|
|
74
|
-
|
|
75
|
-
def scan_filter
|
|
76
|
-
conditions.reduce({}) do |result, (attr, cond)|
|
|
77
|
-
condition = {
|
|
78
|
-
comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
|
|
79
|
-
attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
|
|
80
|
-
}
|
|
81
|
-
# nil means operator doesn't require attribute value list
|
|
82
|
-
conditions.delete(:attribute_value_list) if conditions[:attribute_value_list].nil?
|
|
83
|
-
result[attr] = condition
|
|
84
|
-
result
|
|
85
|
-
end
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
def attributes_to_get
|
|
89
|
-
return if options[:project].nil?
|
|
90
|
-
|
|
91
|
-
options[:project].map(&:to_s)
|
|
92
|
-
end
|
|
93
99
|
end
|
|
94
100
|
end
|
|
95
101
|
end
|