dynamoid 2.2.0 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +53 -0
- data/.rubocop_todo.yml +55 -0
- data/.travis.yml +5 -27
- data/Appraisals +17 -15
- data/CHANGELOG.md +26 -3
- data/Gemfile +4 -2
- data/README.md +95 -77
- data/Rakefile +17 -17
- data/Vagrantfile +5 -3
- data/dynamoid.gemspec +39 -45
- data/gemfiles/rails_4_2.gemfile +7 -5
- data/gemfiles/rails_5_0.gemfile +6 -4
- data/gemfiles/rails_5_1.gemfile +6 -4
- data/gemfiles/rails_5_2.gemfile +6 -4
- data/lib/dynamoid.rb +11 -4
- data/lib/dynamoid/adapter.rb +21 -27
- data/lib/dynamoid/adapter_plugin/{aws_sdk_v2.rb → aws_sdk_v3.rb} +118 -113
- data/lib/dynamoid/application_time_zone.rb +27 -0
- data/lib/dynamoid/associations.rb +3 -6
- data/lib/dynamoid/associations/association.rb +3 -6
- data/lib/dynamoid/associations/belongs_to.rb +4 -5
- data/lib/dynamoid/associations/has_and_belongs_to_many.rb +2 -3
- data/lib/dynamoid/associations/has_many.rb +2 -3
- data/lib/dynamoid/associations/has_one.rb +2 -3
- data/lib/dynamoid/associations/many_association.rb +8 -9
- data/lib/dynamoid/associations/single_association.rb +3 -3
- data/lib/dynamoid/components.rb +2 -2
- data/lib/dynamoid/config.rb +9 -5
- data/lib/dynamoid/config/backoff_strategies/constant_backoff.rb +4 -2
- data/lib/dynamoid/config/backoff_strategies/exponential_backoff.rb +3 -1
- data/lib/dynamoid/config/options.rb +4 -4
- data/lib/dynamoid/criteria.rb +3 -5
- data/lib/dynamoid/criteria/chain.rb +42 -49
- data/lib/dynamoid/dirty.rb +5 -4
- data/lib/dynamoid/document.rb +142 -36
- data/lib/dynamoid/dumping.rb +167 -0
- data/lib/dynamoid/dynamodb_time_zone.rb +16 -0
- data/lib/dynamoid/errors.rb +7 -6
- data/lib/dynamoid/fields.rb +24 -23
- data/lib/dynamoid/finders.rb +101 -59
- data/lib/dynamoid/identity_map.rb +5 -11
- data/lib/dynamoid/indexes.rb +45 -46
- data/lib/dynamoid/middleware/identity_map.rb +2 -0
- data/lib/dynamoid/persistence.rb +67 -307
- data/lib/dynamoid/primary_key_type_mapping.rb +34 -0
- data/lib/dynamoid/railtie.rb +3 -1
- data/lib/dynamoid/tasks/database.rake +11 -11
- data/lib/dynamoid/tasks/database.rb +4 -3
- data/lib/dynamoid/type_casting.rb +193 -0
- data/lib/dynamoid/undumping.rb +188 -0
- data/lib/dynamoid/validations.rb +4 -7
- data/lib/dynamoid/version.rb +3 -1
- metadata +59 -53
- data/gemfiles/rails_4_0.gemfile +0 -9
- data/gemfiles/rails_4_1.gemfile +0 -9
data/Rakefile
CHANGED
@@ -1,32 +1,32 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
|
5
|
+
require 'bundler/setup'
|
4
6
|
begin
|
5
7
|
Bundler.setup(:default, :development)
|
6
8
|
rescue Bundler::BundlerError => e
|
7
|
-
|
8
|
-
|
9
|
+
warn e.message
|
10
|
+
warn 'Run `bundle install` to install missing gems'
|
9
11
|
exit e.status_code
|
10
12
|
end
|
11
|
-
if defined?(Rails)
|
12
|
-
load "./lib/dynamoid/tasks/database.rake"
|
13
|
-
end
|
13
|
+
load './lib/dynamoid/tasks/database.rake' if defined?(Rails)
|
14
14
|
|
15
|
-
require
|
16
|
-
require
|
15
|
+
require 'rake'
|
16
|
+
require 'rspec/core/rake_task'
|
17
17
|
RSpec::Core::RakeTask.new(:spec) do |spec|
|
18
|
-
spec.pattern = FileList[
|
18
|
+
spec.pattern = FileList['spec/**/*_spec.rb']
|
19
19
|
end
|
20
20
|
|
21
|
-
require
|
21
|
+
require 'yard'
|
22
22
|
YARD::Rake::YardocTask.new do |t|
|
23
|
-
t.files = [
|
24
|
-
t.options = [
|
23
|
+
t.files = ['lib/**/*.rb', 'README', 'LICENSE'] # optional
|
24
|
+
t.options = ['-m', 'markdown'] # optional
|
25
25
|
end
|
26
26
|
|
27
|
-
desc
|
27
|
+
desc 'Publish documentation to gh-pages'
|
28
28
|
task :publish do
|
29
|
-
Rake::Task[
|
29
|
+
Rake::Task['yard'].invoke
|
30
30
|
`git add .`
|
31
31
|
`git commit -m 'Regenerated documentation'`
|
32
32
|
`git checkout gh-pages`
|
@@ -41,6 +41,6 @@ task :publish do
|
|
41
41
|
`git checkout master`
|
42
42
|
end
|
43
43
|
|
44
|
-
require
|
44
|
+
require 'wwtd/tasks'
|
45
45
|
|
46
|
-
task :
|
46
|
+
task default: :spec
|
data/Vagrantfile
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
Vagrant.configure('2') do |config|
|
2
4
|
# Choose base box
|
3
5
|
config.vm.box = 'bento/ubuntu-16.04'
|
@@ -16,11 +18,11 @@ Vagrant.configure('2') do |config|
|
|
16
18
|
salt.minion_config = '.dev/vagrant/minion'
|
17
19
|
|
18
20
|
# Pillars
|
19
|
-
salt.pillar(
|
21
|
+
salt.pillar(
|
20
22
|
'ruby' => {
|
21
|
-
'version' => '2.4.1'
|
23
|
+
'version' => '2.4.1'
|
22
24
|
}
|
23
|
-
|
25
|
+
)
|
24
26
|
|
25
27
|
salt.run_highstate = true
|
26
28
|
end
|
data/dynamoid.gemspec
CHANGED
@@ -1,62 +1,56 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
5
|
+
require 'dynamoid/version'
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
8
|
+
spec.name = 'dynamoid'
|
8
9
|
spec.version = Dynamoid::VERSION
|
9
10
|
|
10
11
|
# Keep in sync with README
|
11
12
|
spec.authors = [
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
13
|
+
'Josh Symonds',
|
14
|
+
'Logan Bowers',
|
15
|
+
'Craig Heneveld',
|
16
|
+
'Anatha Kumaran',
|
17
|
+
'Jason Dew',
|
18
|
+
'Luis Arias',
|
19
|
+
'Stefan Neculai',
|
20
|
+
'Philip White',
|
21
|
+
'Peeyush Kumar',
|
22
|
+
'Sumanth Ravipati',
|
23
|
+
'Pascal Corpet',
|
24
|
+
'Brian Glusman',
|
25
|
+
'Peter Boling',
|
26
|
+
'Andrew Konchin'
|
26
27
|
]
|
27
|
-
spec.email = [
|
28
|
+
spec.email = ['peter.boling@gmail.com', 'brian@stellaservice.com']
|
28
29
|
|
29
30
|
spec.description = "Dynamoid is an ORM for Amazon's DynamoDB that supports offline development, associations, querying, and everything else you'd expect from an ActiveRecord-style replacement."
|
30
31
|
spec.summary = "Dynamoid is an ORM for Amazon's DynamoDB"
|
31
32
|
spec.extra_rdoc_files = [
|
32
|
-
|
33
|
-
|
33
|
+
'LICENSE.txt',
|
34
|
+
'README.md'
|
34
35
|
]
|
35
36
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(bin|test|spec|features|.dev|Vagrantfile)/}) }
|
36
|
-
spec.homepage =
|
37
|
-
spec.licenses = [
|
38
|
-
spec.bindir =
|
37
|
+
spec.homepage = 'http://github.com/Dynamoid/Dynamoid'
|
38
|
+
spec.licenses = ['MIT']
|
39
|
+
spec.bindir = 'exe'
|
39
40
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
40
|
-
spec.require_paths = [
|
41
|
+
spec.require_paths = ['lib']
|
42
|
+
|
43
|
+
spec.add_runtime_dependency 'activemodel', '>=4'
|
44
|
+
spec.add_runtime_dependency 'aws-sdk-dynamodb', '~> 1'
|
45
|
+
spec.add_runtime_dependency 'concurrent-ruby', '>= 1.0'
|
41
46
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
spec.
|
51
|
-
spec.add_development_dependency(%q<activesupport>, [">= 4"])
|
52
|
-
spec.add_runtime_dependency(%q<aws-sdk-resources>, ["~> 2"])
|
53
|
-
spec.add_runtime_dependency(%q<concurrent-ruby>, [">= 1.0"])
|
54
|
-
spec.add_development_dependency "pry"
|
55
|
-
spec.add_development_dependency "bundler", "~> 1.14"
|
56
|
-
spec.add_development_dependency "rake", "~> 12.0"
|
57
|
-
spec.add_development_dependency "rspec", "~> 3.0"
|
58
|
-
spec.add_development_dependency "appraisal", "~> 2.1"
|
59
|
-
spec.add_development_dependency "wwtd", "~> 1.3"
|
60
|
-
spec.add_development_dependency(%q<yard>, [">= 0"])
|
61
|
-
spec.add_development_dependency "coveralls", "~> 0.8"
|
47
|
+
spec.add_development_dependency 'appraisal'
|
48
|
+
spec.add_development_dependency 'bundler'
|
49
|
+
spec.add_development_dependency 'coveralls'
|
50
|
+
spec.add_development_dependency 'pry'
|
51
|
+
spec.add_development_dependency 'rake'
|
52
|
+
spec.add_development_dependency 'rspec'
|
53
|
+
spec.add_development_dependency 'rubocop'
|
54
|
+
spec.add_development_dependency 'wwtd'
|
55
|
+
spec.add_development_dependency 'yard'
|
62
56
|
end
|
data/gemfiles/rails_4_2.gemfile
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# This file was generated by Appraisal
|
2
4
|
|
3
|
-
source
|
5
|
+
source 'https://rubygems.org'
|
4
6
|
|
5
|
-
gem
|
6
|
-
gem
|
7
|
-
gem
|
7
|
+
gem 'nokogiri', '~> 1.6.8'
|
8
|
+
gem 'pry-byebug', platforms: :ruby
|
9
|
+
gem 'rails', '~> 4.2.0'
|
8
10
|
|
9
|
-
gemspec path:
|
11
|
+
gemspec path: '../'
|
data/gemfiles/rails_5_0.gemfile
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# This file was generated by Appraisal
|
2
4
|
|
3
|
-
source
|
5
|
+
source 'https://rubygems.org'
|
4
6
|
|
5
|
-
gem
|
6
|
-
gem
|
7
|
+
gem 'pry-byebug', platforms: :ruby
|
8
|
+
gem 'rails', '~> 5.0.0'
|
7
9
|
|
8
|
-
gemspec path:
|
10
|
+
gemspec path: '../'
|
data/gemfiles/rails_5_1.gemfile
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# This file was generated by Appraisal
|
2
4
|
|
3
|
-
source
|
5
|
+
source 'https://rubygems.org'
|
4
6
|
|
5
|
-
gem
|
6
|
-
gem
|
7
|
+
gem 'pry-byebug', platforms: :ruby
|
8
|
+
gem 'rails', '~> 5.1.0'
|
7
9
|
|
8
|
-
gemspec path:
|
10
|
+
gemspec path: '../'
|
data/gemfiles/rails_5_2.gemfile
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# This file was generated by Appraisal
|
2
4
|
|
3
|
-
source
|
5
|
+
source 'https://rubygems.org'
|
4
6
|
|
5
|
-
gem
|
6
|
-
gem
|
7
|
+
gem 'pry-byebug', platforms: :ruby
|
8
|
+
gem 'rails', '~> 5.2.0'
|
7
9
|
|
8
|
-
gemspec path:
|
10
|
+
gemspec path: '../'
|
data/lib/dynamoid.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aws-sdk-dynamodb'
|
1
4
|
require 'delegate'
|
2
5
|
require 'time'
|
3
6
|
require 'securerandom'
|
@@ -11,10 +14,16 @@ require 'active_model'
|
|
11
14
|
|
12
15
|
require 'dynamoid/version'
|
13
16
|
require 'dynamoid/errors'
|
17
|
+
require 'dynamoid/application_time_zone'
|
18
|
+
require 'dynamoid/dynamodb_time_zone'
|
14
19
|
require 'dynamoid/fields'
|
15
20
|
require 'dynamoid/indexes'
|
16
21
|
require 'dynamoid/associations'
|
17
22
|
require 'dynamoid/persistence'
|
23
|
+
require 'dynamoid/dumping'
|
24
|
+
require 'dynamoid/undumping'
|
25
|
+
require 'dynamoid/type_casting'
|
26
|
+
require 'dynamoid/primary_key_type_mapping'
|
18
27
|
require 'dynamoid/dirty'
|
19
28
|
require 'dynamoid/validations'
|
20
29
|
require 'dynamoid/criteria'
|
@@ -29,9 +38,7 @@ require 'dynamoid/tasks/database'
|
|
29
38
|
|
30
39
|
require 'dynamoid/middleware/identity_map'
|
31
40
|
|
32
|
-
if defined?(Rails)
|
33
|
-
require 'dynamoid/railtie'
|
34
|
-
end
|
41
|
+
require 'dynamoid/railtie' if defined?(Rails)
|
35
42
|
|
36
43
|
module Dynamoid
|
37
44
|
extend self
|
@@ -39,7 +46,7 @@ module Dynamoid
|
|
39
46
|
def configure
|
40
47
|
block_given? ? yield(Dynamoid::Config) : Dynamoid::Config
|
41
48
|
end
|
42
|
-
alias
|
49
|
+
alias config configure
|
43
50
|
|
44
51
|
def logger
|
45
52
|
Dynamoid::Config.logger
|
data/lib/dynamoid/adapter.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# require only 'concurrent/atom' once this issue is resolved:
|
2
4
|
# https://github.com/ruby-concurrency/concurrent-ruby/pull/377
|
3
5
|
require 'concurrent'
|
4
6
|
|
5
7
|
# encoding: utf-8
|
6
8
|
module Dynamoid
|
7
|
-
|
8
9
|
# Adapter's value-add:
|
9
10
|
# 1) For the rest of Dynamoid, the gateway to DynamoDB.
|
10
11
|
# 2) Allows switching `config.adapter` to ease development of a new adapter.
|
@@ -16,8 +17,8 @@ module Dynamoid
|
|
16
17
|
end
|
17
18
|
|
18
19
|
def tables
|
19
|
-
|
20
|
-
@tables_.swap{|
|
20
|
+
unless @tables_.value
|
21
|
+
@tables_.swap { |_value, _args| benchmark('Cache Tables') { list_tables || [] } }
|
21
22
|
end
|
22
23
|
@tables_.value
|
23
24
|
end
|
@@ -26,7 +27,7 @@ module Dynamoid
|
|
26
27
|
#
|
27
28
|
# @since 0.2.0
|
28
29
|
def adapter
|
29
|
-
|
30
|
+
unless @adapter_.value
|
30
31
|
adapter = self.class.adapter_plugin_class.new
|
31
32
|
adapter.connect! if adapter.respond_to?(:connect!)
|
32
33
|
@adapter_.compare_and_set(nil, adapter)
|
@@ -36,7 +37,7 @@ module Dynamoid
|
|
36
37
|
end
|
37
38
|
|
38
39
|
def clear_cache!
|
39
|
-
@tables_.swap{|
|
40
|
+
@tables_.swap { |_value, _args| nil }
|
40
41
|
end
|
41
42
|
|
42
43
|
# Shows how long it takes a method to run on the adapter. Useful for generating logged output.
|
@@ -51,8 +52,8 @@ module Dynamoid
|
|
51
52
|
def benchmark(method, *args)
|
52
53
|
start = Time.now
|
53
54
|
result = yield
|
54
|
-
Dynamoid.logger.debug "(#{((Time.now - start) * 1000.0).round(2)} ms) #{method.to_s.split('_').collect(&:upcase).join(' ')}#{
|
55
|
-
|
55
|
+
Dynamoid.logger.debug "(#{((Time.now - start) * 1000.0).round(2)} ms) #{method.to_s.split('_').collect(&:upcase).join(' ')}#{" - #{args.inspect}" unless args.nil? || args.empty?}"
|
56
|
+
result
|
56
57
|
end
|
57
58
|
|
58
59
|
# Write an object to the adapter.
|
@@ -81,13 +82,9 @@ module Dynamoid
|
|
81
82
|
#
|
82
83
|
# @since 0.2.0
|
83
84
|
def read(table, ids, options = {}, &blk)
|
84
|
-
range_key = options.delete(:range_key)
|
85
|
-
|
86
85
|
if ids.respond_to?(:each)
|
87
|
-
|
88
|
-
batch_get_item({table => ids}, options, &blk)
|
86
|
+
batch_get_item({ table => ids }, options, &blk)
|
89
87
|
else
|
90
|
-
options[:range_key] = range_key if range_key
|
91
88
|
get_item(table, ids, options)
|
92
89
|
end
|
93
90
|
end
|
@@ -101,12 +98,12 @@ module Dynamoid
|
|
101
98
|
def delete(table, ids, options = {})
|
102
99
|
range_key = options[:range_key] # array of range keys that matches the ids passed in
|
103
100
|
if ids.respond_to?(:each)
|
104
|
-
if range_key.respond_to?(:each)
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
101
|
+
ids = if range_key.respond_to?(:each)
|
102
|
+
# turn ids into array of arrays each element being hash_key, range_key
|
103
|
+
ids.each_with_index.map { |id, i| [id, range_key[i]] }
|
104
|
+
else
|
105
|
+
range_key ? ids.map { |id| [id, range_key] } : ids
|
106
|
+
end
|
110
107
|
|
111
108
|
batch_delete_item(table => ids)
|
112
109
|
else
|
@@ -121,11 +118,11 @@ module Dynamoid
|
|
121
118
|
#
|
122
119
|
# @since 0.2.0
|
123
120
|
def scan(table, query = {}, opts = {})
|
124
|
-
benchmark('Scan', table, query) {adapter.scan(table, query, opts)}
|
121
|
+
benchmark('Scan', table, query) { adapter.scan(table, query, opts) }
|
125
122
|
end
|
126
123
|
|
127
124
|
def create_table(table_name, key, options = {})
|
128
|
-
|
125
|
+
unless tables.include?(table_name)
|
129
126
|
benchmark('Create Table') { adapter.create_table(table_name, key, options) }
|
130
127
|
tables << table_name
|
131
128
|
end
|
@@ -140,15 +137,15 @@ module Dynamoid
|
|
140
137
|
end
|
141
138
|
end
|
142
139
|
|
143
|
-
[
|
140
|
+
%i[batch_get_item delete_item get_item list_tables put_item truncate batch_write_item batch_delete_item].each do |m|
|
144
141
|
# Method delegation with benchmark to the underlying adapter. Faster than relying on method_missing.
|
145
142
|
#
|
146
143
|
# @since 0.2.0
|
147
144
|
define_method(m) do |*args, &blk|
|
148
145
|
if blk.present?
|
149
|
-
benchmark(
|
146
|
+
benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
|
150
147
|
else
|
151
|
-
benchmark(
|
148
|
+
benchmark(m.to_s, *args) { adapter.send(m, *args) }
|
152
149
|
end
|
153
150
|
end
|
154
151
|
end
|
@@ -157,7 +154,7 @@ module Dynamoid
|
|
157
154
|
#
|
158
155
|
# @since 0.2.0
|
159
156
|
def method_missing(method, *args, &block)
|
160
|
-
return benchmark(method, *args) {adapter.send(method, *args, &block)} if adapter.respond_to?(method)
|
157
|
+
return benchmark(method, *args) { adapter.send(method, *args, &block) } if adapter.respond_to?(method)
|
161
158
|
super
|
162
159
|
end
|
163
160
|
|
@@ -180,8 +177,6 @@ module Dynamoid
|
|
180
177
|
adapter.query(table_name, opts)
|
181
178
|
end
|
182
179
|
|
183
|
-
private
|
184
|
-
|
185
180
|
def self.adapter_plugin_class
|
186
181
|
unless Dynamoid.const_defined?(:AdapterPlugin) && Dynamoid::AdapterPlugin.const_defined?(Dynamoid::Config.adapter.camelcase)
|
187
182
|
require "dynamoid/adapter_plugin/#{Dynamoid::Config.adapter}"
|
@@ -189,6 +184,5 @@ module Dynamoid
|
|
189
184
|
|
190
185
|
Dynamoid::AdapterPlugin.const_get(Dynamoid::Config.adapter.camelcase)
|
191
186
|
end
|
192
|
-
|
193
187
|
end
|
194
188
|
end
|
@@ -1,46 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Dynamoid
|
2
4
|
module AdapterPlugin
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
EQ = 'EQ'.freeze
|
5
|
+
# The AwsSdkV3 adapter provides support for the aws-sdk version 2 for ruby.
|
6
|
+
class AwsSdkV3
|
7
|
+
EQ = 'EQ'
|
7
8
|
RANGE_MAP = {
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
}
|
9
|
+
range_greater_than: 'GT',
|
10
|
+
range_less_than: 'LT',
|
11
|
+
range_gte: 'GE',
|
12
|
+
range_lte: 'LE',
|
13
|
+
range_begins_with: 'BEGINS_WITH',
|
14
|
+
range_between: 'BETWEEN',
|
15
|
+
range_eq: 'EQ'
|
16
|
+
}.freeze
|
16
17
|
|
17
18
|
# Don't implement NULL and NOT_NULL because it doesn't make seanse -
|
18
19
|
# we declare schema in models
|
19
20
|
FIELD_MAP = {
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
}
|
32
|
-
HASH_KEY = 'HASH'
|
33
|
-
RANGE_KEY = 'RANGE'
|
34
|
-
STRING_TYPE = 'S'
|
35
|
-
NUM_TYPE = 'N'
|
36
|
-
BINARY_TYPE = 'B'
|
21
|
+
eq: 'EQ',
|
22
|
+
ne: 'NE',
|
23
|
+
gt: 'GT',
|
24
|
+
lt: 'LT',
|
25
|
+
gte: 'GE',
|
26
|
+
lte: 'LE',
|
27
|
+
begins_with: 'BEGINS_WITH',
|
28
|
+
between: 'BETWEEN',
|
29
|
+
in: 'IN',
|
30
|
+
contains: 'CONTAINS',
|
31
|
+
not_contains: 'NOT_CONTAINS'
|
32
|
+
}.freeze
|
33
|
+
HASH_KEY = 'HASH'
|
34
|
+
RANGE_KEY = 'RANGE'
|
35
|
+
STRING_TYPE = 'S'
|
36
|
+
NUM_TYPE = 'N'
|
37
|
+
BINARY_TYPE = 'B'
|
37
38
|
TABLE_STATUSES = {
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
39
|
+
creating: 'CREATING',
|
40
|
+
updating: 'UPDATING',
|
41
|
+
deleting: 'DELETING',
|
42
|
+
active: 'ACTIVE'
|
42
43
|
}.freeze
|
43
|
-
PARSE_TABLE_STATUS =
|
44
|
+
PARSE_TABLE_STATUS = lambda { |resp, lookup = :table|
|
44
45
|
# lookup is table for describe_table API
|
45
46
|
# lookup is table_description for create_table API
|
46
47
|
# because Amazon, damnit.
|
@@ -74,6 +75,13 @@ module Dynamoid
|
|
74
75
|
@connection_hash[:region] = Dynamoid::Config.region
|
75
76
|
end
|
76
77
|
|
78
|
+
# https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/logging.rb
|
79
|
+
# https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb
|
80
|
+
formatter = Aws::Log::Formatter.new(':operation | Request :http_request_body | Response :http_response_body')
|
81
|
+
@connection_hash[:logger] = Dynamoid::Config.logger
|
82
|
+
@connection_hash[:log_level] = :debug
|
83
|
+
@connection_hash[:log_formatter] = formatter
|
84
|
+
|
77
85
|
@connection_hash
|
78
86
|
end
|
79
87
|
|
@@ -90,10 +98,10 @@ module Dynamoid
|
|
90
98
|
# Block receives boolean flag which is true if there are some unprocessed items, otherwise false.
|
91
99
|
#
|
92
100
|
# @example Saves several items to the table testtable
|
93
|
-
# Dynamoid::AdapterPlugin::
|
101
|
+
# Dynamoid::AdapterPlugin::AwsSdkV3.batch_write_item('table1', [{ id: '1', name: 'a' }, { id: '2', name: 'b'}])
|
94
102
|
#
|
95
103
|
# @example Pass block
|
96
|
-
# Dynamoid::AdapterPlugin::
|
104
|
+
# Dynamoid::AdapterPlugin::AwsSdkV3.batch_write_item('table1', items) do |bool|
|
97
105
|
# if bool
|
98
106
|
# puts 'there are unprocessed items'
|
99
107
|
# end
|
@@ -107,27 +115,25 @@ module Dynamoid
|
|
107
115
|
# See:
|
108
116
|
# * http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
|
109
117
|
# * http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
|
110
|
-
def batch_write_item
|
118
|
+
def batch_write_item(table_name, objects, options = {})
|
111
119
|
items = objects.map { |o| sanitize_item(o) }
|
112
120
|
|
113
121
|
begin
|
114
|
-
while items.present?
|
122
|
+
while items.present?
|
115
123
|
batch = items.shift(BATCH_WRITE_ITEM_REQUESTS_LIMIT)
|
116
124
|
requests = batch.map { |item| { put_request: { item: item } } }
|
117
125
|
|
118
126
|
response = client.batch_write_item(
|
119
127
|
{
|
120
128
|
request_items: {
|
121
|
-
table_name => requests
|
129
|
+
table_name => requests
|
122
130
|
},
|
123
131
|
return_consumed_capacity: 'TOTAL',
|
124
132
|
return_item_collection_metrics: 'SIZE'
|
125
133
|
}.merge!(options)
|
126
134
|
)
|
127
135
|
|
128
|
-
if block_given?
|
129
|
-
yield(response.unprocessed_items.present?)
|
130
|
-
end
|
136
|
+
yield(response.unprocessed_items.present?) if block_given?
|
131
137
|
|
132
138
|
if response.unprocessed_items.present?
|
133
139
|
items += response.unprocessed_items[table_name].map { |r| r.put_request.item }
|
@@ -148,10 +154,10 @@ module Dynamoid
|
|
148
154
|
# * and boolean flag is true if there are some unprocessed keys, otherwise false.
|
149
155
|
#
|
150
156
|
# @example Retrieve IDs 1 and 2 from the table testtable
|
151
|
-
# Dynamoid::AdapterPlugin::
|
157
|
+
# Dynamoid::AdapterPlugin::AwsSdkV3.batch_get_item('table1' => ['1', '2'])
|
152
158
|
#
|
153
159
|
# @example Pass block to receive each batch
|
154
|
-
# Dynamoid::AdapterPlugin::
|
160
|
+
# Dynamoid::AdapterPlugin::AwsSdkV3.batch_get_item('table1' => ids) do |hash, bool|
|
155
161
|
# puts hash['table1']
|
156
162
|
#
|
157
163
|
# if bool
|
@@ -171,9 +177,9 @@ module Dynamoid
|
|
171
177
|
# @since 1.0.0
|
172
178
|
#
|
173
179
|
# @todo: Provide support for passing options to underlying batch_get_item
|
174
|
-
def batch_get_item(table_ids,
|
175
|
-
request_items = Hash.new{|h, k| h[k] = []}
|
176
|
-
return request_items if table_ids.all?{|
|
180
|
+
def batch_get_item(table_ids, _options = {})
|
181
|
+
request_items = Hash.new { |h, k| h[k] = [] }
|
182
|
+
return request_items if table_ids.all? { |_k, v| v.blank? }
|
177
183
|
|
178
184
|
ret = Hash.new([].freeze) # Default for tables where no rows are returned
|
179
185
|
|
@@ -184,20 +190,20 @@ module Dynamoid
|
|
184
190
|
hk = tbl.hash_key.to_s
|
185
191
|
rng = tbl.range_key.to_s
|
186
192
|
|
187
|
-
while ids.present?
|
193
|
+
while ids.present?
|
188
194
|
batch = ids.shift(Dynamoid::Config.batch_size)
|
189
195
|
|
190
|
-
request_items = Hash.new{|h, k| h[k] = []}
|
196
|
+
request_items = Hash.new { |h, k| h[k] = [] }
|
191
197
|
|
192
198
|
keys = if rng.present?
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
199
|
+
Array(batch).map do |h, r|
|
200
|
+
{ hk => h, rng => r }
|
201
|
+
end
|
202
|
+
else
|
203
|
+
Array(batch).map do |id|
|
204
|
+
{ hk => id }
|
205
|
+
end
|
206
|
+
end
|
201
207
|
|
202
208
|
request_items[t] = {
|
203
209
|
keys: keys
|
@@ -207,11 +213,7 @@ module Dynamoid
|
|
207
213
|
request_items: request_items
|
208
214
|
)
|
209
215
|
|
210
|
-
|
211
|
-
results.data[:responses].each do |table, rows|
|
212
|
-
ret[table] += rows.collect { |r| result_item_to_hash(r) }
|
213
|
-
end
|
214
|
-
else
|
216
|
+
if block_given?
|
215
217
|
batch_results = Hash.new([].freeze)
|
216
218
|
|
217
219
|
results.data[:responses].each do |table, rows|
|
@@ -219,6 +221,10 @@ module Dynamoid
|
|
219
221
|
end
|
220
222
|
|
221
223
|
yield(batch_results, results.unprocessed_keys.present?)
|
224
|
+
else
|
225
|
+
results.data[:responses].each do |table, rows|
|
226
|
+
ret[table] += rows.collect { |r| result_item_to_hash(r) }
|
227
|
+
end
|
222
228
|
end
|
223
229
|
|
224
230
|
if results.unprocessed_keys.present?
|
@@ -227,9 +233,7 @@ module Dynamoid
|
|
227
233
|
end
|
228
234
|
end
|
229
235
|
|
230
|
-
unless block_given?
|
231
|
-
ret
|
232
|
-
end
|
236
|
+
ret unless block_given?
|
233
237
|
end
|
234
238
|
|
235
239
|
# Delete many items at once from DynamoDB. More efficient than delete each item individually.
|
@@ -237,7 +241,7 @@ module Dynamoid
|
|
237
241
|
# @example Delete IDs 1 and 2 from the table testtable
|
238
242
|
# Dynamoid::AdapterPlugin::AwsSdk.batch_delete_item('table1' => ['1', '2'])
|
239
243
|
# or
|
240
|
-
# Dynamoid::AdapterPlugin::
|
244
|
+
# Dynamoid::AdapterPlugin::AwsSdkV3.batch_delete_item('table1' => [['hk1', 'rk2'], ['hk1', 'rk2']]]))
|
241
245
|
#
|
242
246
|
# @param [Hash] options the hash of tables and IDs to delete
|
243
247
|
#
|
@@ -253,11 +257,11 @@ module Dynamoid
|
|
253
257
|
table = describe_table(table_name)
|
254
258
|
|
255
259
|
ids.each_slice(BATCH_WRITE_ITEM_REQUESTS_LIMIT) do |sliced_ids|
|
256
|
-
delete_requests = sliced_ids.map
|
257
|
-
{delete_request: {key: key_stanza(table, *id)}}
|
258
|
-
|
260
|
+
delete_requests = sliced_ids.map do |id|
|
261
|
+
{ delete_request: { key: key_stanza(table, *id) } }
|
262
|
+
end
|
259
263
|
|
260
|
-
requests << {table_name => delete_requests}
|
264
|
+
requests << { table_name => delete_requests }
|
261
265
|
end
|
262
266
|
end
|
263
267
|
|
@@ -266,7 +270,8 @@ module Dynamoid
|
|
266
270
|
client.batch_write_item(
|
267
271
|
request_items: request_items,
|
268
272
|
return_consumed_capacity: 'TOTAL',
|
269
|
-
return_item_collection_metrics: 'SIZE'
|
273
|
+
return_item_collection_metrics: 'SIZE'
|
274
|
+
)
|
270
275
|
end
|
271
276
|
rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
|
272
277
|
raise Dynamoid::Errors::ConditionalCheckFailedException, e
|
@@ -330,10 +335,10 @@ module Dynamoid
|
|
330
335
|
end
|
331
336
|
end
|
332
337
|
resp = client.create_table(client_opts)
|
333
|
-
options[:sync] = true if !options.
|
338
|
+
options[:sync] = true if !options.key?(:sync) && ls_indexes.present? || gs_indexes.present?
|
334
339
|
until_past_table_status(table_name, :creating) if options[:sync] &&
|
335
|
-
|
336
|
-
|
340
|
+
(status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
|
341
|
+
status == TABLE_STATUSES[:creating]
|
337
342
|
# Response to original create_table, which, if options[:sync]
|
338
343
|
# may have an outdated table_description.table_status of "CREATING"
|
339
344
|
resp
|
@@ -395,8 +400,8 @@ module Dynamoid
|
|
395
400
|
def delete_table(table_name, options = {})
|
396
401
|
resp = client.delete_table(table_name: table_name)
|
397
402
|
until_past_table_status(table_name, :deleting) if options[:sync] &&
|
398
|
-
|
399
|
-
|
403
|
+
(status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
|
404
|
+
status == TABLE_STATUSES[:deleting]
|
400
405
|
table_cache.delete(table_name)
|
401
406
|
rescue Aws::DynamoDB::Errors::ResourceInUseException => e
|
402
407
|
Dynamoid.logger.error "Table #{table_name} cannot be deleted as it is in use"
|
@@ -422,12 +427,11 @@ module Dynamoid
|
|
422
427
|
# @todo Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#get_item-instance_method
|
423
428
|
def get_item(table_name, key, options = {})
|
424
429
|
options ||= {}
|
425
|
-
table
|
430
|
+
table = describe_table(table_name)
|
426
431
|
range_key = options.delete(:range_key)
|
427
432
|
|
428
433
|
item = client.get_item(table_name: table_name,
|
429
|
-
|
430
|
-
)[:item]
|
434
|
+
key: key_stanza(table, key, range_key))[:item]
|
431
435
|
item ? result_item_to_hash(item) : nil
|
432
436
|
end
|
433
437
|
|
@@ -450,11 +454,10 @@ module Dynamoid
|
|
450
454
|
raise "non-empty options: #{options}" unless options.empty?
|
451
455
|
begin
|
452
456
|
result = client.update_item(table_name: table_name,
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
)
|
457
|
+
key: key_stanza(table, key, range_key),
|
458
|
+
attribute_updates: iu.to_h,
|
459
|
+
expected: expected_stanza(conditions),
|
460
|
+
return_values: 'ALL_NEW')
|
458
461
|
result_item_to_hash(result[:attributes])
|
459
462
|
rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
|
460
463
|
raise Dynamoid::Errors::ConditionalCheckFailedException, e
|
@@ -518,11 +521,11 @@ module Dynamoid
|
|
518
521
|
hk = (opts[:hash_key].present? ? opts.delete(:hash_key) : table.hash_key).to_s
|
519
522
|
rng = (opts[:range_key].present? ? opts.delete(:range_key) : table.range_key).to_s
|
520
523
|
q = opts.slice(
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
524
|
+
:consistent_read,
|
525
|
+
:scan_index_forward,
|
526
|
+
:select,
|
527
|
+
:index_name
|
528
|
+
)
|
526
529
|
|
527
530
|
opts.delete(:consistent_read)
|
528
531
|
opts.delete(:scan_index_forward)
|
@@ -534,7 +537,7 @@ module Dynamoid
|
|
534
537
|
scan_limit = opts.delete(:scan_limit)
|
535
538
|
batch_size = opts.delete(:batch_size)
|
536
539
|
exclusive_start_key = opts.delete(:exclusive_start_key)
|
537
|
-
limit = [record_limit, scan_limit, batch_size].compact.min
|
540
|
+
limit = [record_limit, scan_limit, batch_size].compact.min
|
538
541
|
|
539
542
|
key_conditions = {
|
540
543
|
hk => {
|
@@ -543,8 +546,8 @@ module Dynamoid
|
|
543
546
|
}
|
544
547
|
}
|
545
548
|
|
546
|
-
opts.each_pair do |k,
|
547
|
-
next unless(op = RANGE_MAP[k])
|
549
|
+
opts.each_pair do |k, _v|
|
550
|
+
next unless (op = RANGE_MAP[k])
|
548
551
|
key_conditions[rng] = {
|
549
552
|
comparison_operator: op,
|
550
553
|
attribute_value_list: attribute_value_list(op, opts.delete(k).freeze)
|
@@ -552,7 +555,7 @@ module Dynamoid
|
|
552
555
|
end
|
553
556
|
|
554
557
|
query_filter = {}
|
555
|
-
opts.reject {|k, _| k.in? RANGE_MAP.keys}.each do |attr, hash|
|
558
|
+
opts.reject { |k, _| k.in? RANGE_MAP.keys }.each do |attr, hash|
|
556
559
|
query_filter[attr] = {
|
557
560
|
comparison_operator: FIELD_MAP[hash.keys[0]],
|
558
561
|
attribute_value_list: attribute_value_list(FIELD_MAP[hash.keys[0]], hash.values[0].freeze)
|
@@ -565,7 +568,7 @@ module Dynamoid
|
|
565
568
|
q[:key_conditions] = key_conditions
|
566
569
|
q[:query_filter] = query_filter
|
567
570
|
|
568
|
-
Enumerator.new
|
571
|
+
Enumerator.new do |y|
|
569
572
|
record_count = 0
|
570
573
|
scan_count = 0
|
571
574
|
loop do
|
@@ -600,13 +603,13 @@ module Dynamoid
|
|
600
603
|
scan_count += results.scanned_count
|
601
604
|
break if scan_limit && scan_count >= scan_limit
|
602
605
|
|
603
|
-
if(lk = results.last_evaluated_key)
|
606
|
+
if (lk = results.last_evaluated_key)
|
604
607
|
q[:exclusive_start_key] = lk
|
605
608
|
else
|
606
609
|
break
|
607
610
|
end
|
608
611
|
end
|
609
|
-
|
612
|
+
end
|
610
613
|
end
|
611
614
|
|
612
615
|
# Scan the DynamoDB table. This is usually a very slow operation as it naively filters all data on
|
@@ -632,13 +635,13 @@ module Dynamoid
|
|
632
635
|
request_limit = [record_limit, scan_limit, batch_size].compact.min
|
633
636
|
request[:limit] = request_limit if request_limit
|
634
637
|
request[:exclusive_start_key] = exclusive_start_key if exclusive_start_key
|
635
|
-
|
638
|
+
|
636
639
|
if scan_hash.present?
|
637
640
|
request[:scan_filter] = scan_hash.reduce({}) do |memo, (attr, cond)|
|
638
641
|
memo.merge(attr.to_s => {
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
+
comparison_operator: FIELD_MAP[cond.keys[0]],
|
643
|
+
attribute_value_list: attribute_value_list(FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
|
644
|
+
})
|
642
645
|
end
|
643
646
|
end
|
644
647
|
|
@@ -678,7 +681,7 @@ module Dynamoid
|
|
678
681
|
break if scan_limit && scan_count >= scan_limit
|
679
682
|
|
680
683
|
# Keep pulling if we haven't finished paging in all data
|
681
|
-
if(lk = results[:last_evaluated_key])
|
684
|
+
if (lk = results[:last_evaluated_key])
|
682
685
|
request[:exclusive_start_key] = lk
|
683
686
|
else
|
684
687
|
break
|
@@ -715,14 +718,14 @@ module Dynamoid
|
|
715
718
|
status = PARSE_TABLE_STATUS.call(resp)
|
716
719
|
again = counter < Dynamoid::Config.sync_retry_max_times &&
|
717
720
|
status == TABLE_STATUSES[expect_status]
|
718
|
-
{again: again, status: status, counter: counter}
|
721
|
+
{ again: again, status: status, counter: counter }
|
719
722
|
end
|
720
723
|
|
721
724
|
def until_past_table_status(table_name, status = :creating)
|
722
725
|
counter = 0
|
723
726
|
resp = nil
|
724
727
|
begin
|
725
|
-
check = {again: true}
|
728
|
+
check = { again: true }
|
726
729
|
while check[:again]
|
727
730
|
sleep Dynamoid::Config.sync_retry_wait_seconds
|
728
731
|
resp = client.describe_table(table_name: table_name)
|
@@ -756,7 +759,7 @@ module Dynamoid
|
|
756
759
|
# Converts from symbol to the API string for the given data type
|
757
760
|
# E.g. :number -> 'N'
|
758
761
|
def api_type(type)
|
759
|
-
case
|
762
|
+
case type
|
760
763
|
when :string then STRING_TYPE
|
761
764
|
when :number then NUM_TYPE
|
762
765
|
when :binary then BINARY_TYPE
|
@@ -967,7 +970,7 @@ module Dynamoid
|
|
967
970
|
def attribute_value_list(operator, value)
|
968
971
|
# For BETWEEN and IN operators we should keep value as is (it should be already an array)
|
969
972
|
# For all the other operators we wrap the value with array
|
970
|
-
if [
|
973
|
+
if %w[BETWEEN IN].include?(operator)
|
971
974
|
[value].flatten
|
972
975
|
else
|
973
976
|
[value]
|
@@ -993,13 +996,13 @@ module Dynamoid
|
|
993
996
|
end
|
994
997
|
|
995
998
|
def range_type
|
996
|
-
range_type ||= schema[:attribute_definitions].find
|
999
|
+
range_type ||= schema[:attribute_definitions].find do |d|
|
997
1000
|
d[:attribute_name] == range_key
|
998
|
-
|
1001
|
+
end.try(:fetch, :attribute_type, nil)
|
999
1002
|
end
|
1000
1003
|
|
1001
1004
|
def hash_key
|
1002
|
-
@hash_key ||= schema[:key_schema].find { |d| d[:key_type] == HASH_KEY
|
1005
|
+
@hash_key ||= schema[:key_schema].find { |d| d[:key_type] == HASH_KEY }.try(:attribute_name).to_sym
|
1003
1006
|
end
|
1004
1007
|
|
1005
1008
|
#
|
@@ -1024,7 +1027,9 @@ module Dynamoid
|
|
1024
1027
|
attr_reader :table, :key, :range_key
|
1025
1028
|
|
1026
1029
|
def initialize(table, key, range_key = nil)
|
1027
|
-
@table = table
|
1030
|
+
@table = table
|
1031
|
+
@key = key
|
1032
|
+
@range_key = range_key
|
1028
1033
|
@additions = {}
|
1029
1034
|
@deletions = {}
|
1030
1035
|
@updates = {}
|
@@ -1086,13 +1091,13 @@ module Dynamoid
|
|
1086
1091
|
ret
|
1087
1092
|
end
|
1088
1093
|
|
1089
|
-
ADD = 'ADD'
|
1090
|
-
DELETE = 'DELETE'
|
1091
|
-
PUT = 'PUT'
|
1094
|
+
ADD = 'ADD'
|
1095
|
+
DELETE = 'DELETE'
|
1096
|
+
PUT = 'PUT'
|
1092
1097
|
end
|
1093
1098
|
|
1094
1099
|
def sanitize_item(attributes)
|
1095
|
-
attributes.reject do |
|
1100
|
+
attributes.reject do |_k, v|
|
1096
1101
|
v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
|
1097
1102
|
end
|
1098
1103
|
end
|