dynamoid 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Dynamoid.gemspec +5 -3
- data/README.markdown +13 -4
- data/VERSION +1 -1
- data/lib/dynamoid/adapter.rb +6 -4
- data/lib/dynamoid/adapter/aws_sdk.rb +29 -8
- data/lib/dynamoid/adapter/local.rb +37 -12
- data/lib/dynamoid/associations/has_one.rb +10 -2
- data/lib/dynamoid/criteria.rb +1 -1
- data/lib/dynamoid/criteria/chain.rb +42 -6
- data/lib/dynamoid/errors.rb +1 -1
- data/lib/dynamoid/indexes.rb +18 -30
- data/lib/dynamoid/indexes/index.rb +62 -0
- data/lib/dynamoid/persistence.rb +7 -3
- data/spec/app/models/user.rb +4 -0
- data/spec/dynamoid/adapter/aws_sdk_spec.rb +63 -8
- data/spec/dynamoid/adapter/local_spec.rb +89 -5
- data/spec/dynamoid/adapter_spec.rb +29 -1
- data/spec/dynamoid/criteria/chain_spec.rb +31 -8
- data/spec/dynamoid/criteria_spec.rb +1 -1
- data/spec/dynamoid/indexes/index_spec.rb +84 -0
- data/spec/dynamoid/indexes_spec.rb +9 -37
- metadata +69 -22
data/Dynamoid.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "dynamoid"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.2.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Josh Symonds"]
|
12
|
-
s.date = "2012-03-
|
12
|
+
s.date = "2012-03-15"
|
13
13
|
s.description = "Dynamoid is an ORM for Amazon's DynamoDB that supports offline development, associations, querying, and everything else you'd expect from an ActiveRecord-style replacement."
|
14
14
|
s.email = "josh@joshsymonds.com"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -46,6 +46,7 @@ Gem::Specification.new do |s|
|
|
46
46
|
"lib/dynamoid/fields.rb",
|
47
47
|
"lib/dynamoid/finders.rb",
|
48
48
|
"lib/dynamoid/indexes.rb",
|
49
|
+
"lib/dynamoid/indexes/index.rb",
|
49
50
|
"lib/dynamoid/persistence.rb",
|
50
51
|
"spec/app/models/address.rb",
|
51
52
|
"spec/app/models/magazine.rb",
|
@@ -67,6 +68,7 @@ Gem::Specification.new do |s|
|
|
67
68
|
"spec/dynamoid/document_spec.rb",
|
68
69
|
"spec/dynamoid/fields_spec.rb",
|
69
70
|
"spec/dynamoid/finders_spec.rb",
|
71
|
+
"spec/dynamoid/indexes/index_spec.rb",
|
70
72
|
"spec/dynamoid/indexes_spec.rb",
|
71
73
|
"spec/dynamoid/persistence_spec.rb",
|
72
74
|
"spec/dynamoid_spec.rb",
|
@@ -75,7 +77,7 @@ Gem::Specification.new do |s|
|
|
75
77
|
s.homepage = "http://github.com/Veraticus/Dynamoid"
|
76
78
|
s.licenses = ["MIT"]
|
77
79
|
s.require_paths = ["lib"]
|
78
|
-
s.rubygems_version = "1.8.
|
80
|
+
s.rubygems_version = "1.8.18"
|
79
81
|
s.summary = "Dynamoid is an ORM for Amazon's DynamoDB"
|
80
82
|
|
81
83
|
if s.respond_to? :specification_version then
|
data/README.markdown
CHANGED
@@ -6,10 +6,6 @@ DynamoDB is not like other document-based databases you might know, and is very
|
|
6
6
|
|
7
7
|
But if you want a fast, scalable, simple, easy-to-use database (and a Gem that supports it) then look no further!
|
8
8
|
|
9
|
-
## Warning!
|
10
|
-
|
11
|
-
I'm still working on this gem a lot. It only provides .where(arguments) in its criteria chaining so far. More is coming though!
|
12
|
-
|
13
9
|
## Installation
|
14
10
|
|
15
11
|
Installing Dynamoid is pretty simple. First include the Gem in your Gemfile:
|
@@ -48,6 +44,10 @@ class User
|
|
48
44
|
index :name # Only specify indexes if you intend to perform queries on the specified fields.
|
49
45
|
index :email # Fields without indexes suffer extremely poor performance as they must use
|
50
46
|
index [:name, :email] # scan rather than query.
|
47
|
+
index :created_at, :range => true
|
48
|
+
index :name, :range => :created_at
|
49
|
+
# You can only provide one range query for each index, or specify an index
|
50
|
+
# to be only a range query with :range => true.
|
51
51
|
|
52
52
|
has_many :addresses # Associations do not accept any options presently. The referenced
|
53
53
|
# model name must match exactly and the foreign key is always id.
|
@@ -100,6 +100,15 @@ u.addresses.where(:city => 'Chicago').all
|
|
100
100
|
|
101
101
|
But keep in mind Dynamoid -- and document-based storage systems in general -- are not drop-in replacements for existing relational databases. The above query does not efficiently perform a conditional join, but instead finds all the user's addresses and naively filters them in Ruby. For large associations this is a performance hit compared to relational database engines.
|
102
102
|
|
103
|
+
If you have a range index, Dynamoid provides a number of additional other convenience methods to make your life a little easier:
|
104
|
+
|
105
|
+
```ruby
|
106
|
+
User.where("created_at.gt" => DateTime.now - 1.day).all
|
107
|
+
User.where("created_at.lt" => DateTime.now - 1.day).all
|
108
|
+
```
|
109
|
+
|
110
|
+
It also supports .gte and .lte. Turning those into symbols and allowing a Rails SQL-style string syntax is in the works. You can only have one range argument per query, because of DynamoDB's inherent limitations, so use it sensibly!
|
111
|
+
|
103
112
|
## Partitioning, Provisioning, and Performance
|
104
113
|
|
105
114
|
DynamoDB achieves much of its speed by relying on a random pattern of writes and reads: internally, hash keys are distributed across servers, and reading from two consecutive servers is much faster than reading from the same server twice. Of course, many of our applications request one key (like a commonly used role, a superuser, or a very popular product) much more frequently than other keys. In DynamoDB, this will result in lowered throughput and slower response times, and is a design pattern we should try to avoid.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
data/lib/dynamoid/adapter.rb
CHANGED
@@ -32,8 +32,9 @@ module Dynamoid #:nodoc:
|
|
32
32
|
benchmark('Put Item', object) {put_item(table, object)}
|
33
33
|
end
|
34
34
|
|
35
|
-
def read(table, ids)
|
35
|
+
def read(table, ids, range_key = nil)
|
36
36
|
if ids.respond_to?(:each)
|
37
|
+
ids = ids.collect{|id| range_key ? [id, range_key] : id}
|
37
38
|
if Dynamoid::Config.partitioning?
|
38
39
|
results = benchmark('Partitioned Batch Get Item', ids) {batch_get_item(table => id_with_partitions(ids))}
|
39
40
|
{table => result_for_partition(results[table])}
|
@@ -42,10 +43,11 @@ module Dynamoid #:nodoc:
|
|
42
43
|
end
|
43
44
|
else
|
44
45
|
if Dynamoid::Config.partitioning?
|
46
|
+
ids = range_key ? [[ids, range_key]] : ids
|
45
47
|
results = benchmark('Partitioned Get Item', ids) {batch_get_item(table => id_with_partitions(ids))}
|
46
48
|
result_for_partition(results[table]).first
|
47
49
|
else
|
48
|
-
benchmark('Get Item', ids) {get_item(table, ids)}
|
50
|
+
benchmark('Get Item', ids) {get_item(table, ids, range_key)}
|
49
51
|
end
|
50
52
|
end
|
51
53
|
end
|
@@ -69,14 +71,14 @@ module Dynamoid #:nodoc:
|
|
69
71
|
end
|
70
72
|
end
|
71
73
|
|
72
|
-
[:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item
|
74
|
+
[:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item].each do |m|
|
73
75
|
define_method(m) do |*args|
|
74
76
|
benchmark("#{m.to_s}", args) {adapter.send(m, *args)}
|
75
77
|
end
|
76
78
|
end
|
77
79
|
|
78
80
|
def id_with_partitions(ids)
|
79
|
-
Array(ids).collect {|id| (0...Dynamoid::Config.partition_size).collect{|n| "#{id}.#{n}"}}.flatten
|
81
|
+
Array(ids).collect {|id| (0...Dynamoid::Config.partition_size).collect{|n| id.is_a?(Array) ? ["#{id.first}.#{n}", id.last] : "#{id}.#{n}"}}.flatten(1)
|
80
82
|
end
|
81
83
|
|
82
84
|
def result_for_partition(results)
|
@@ -31,17 +31,23 @@ module Dynamoid
|
|
31
31
|
end
|
32
32
|
|
33
33
|
# CreateTable
|
34
|
-
def create_table(table_name, key)
|
35
|
-
|
34
|
+
def create_table(table_name, key = :id, options = {})
|
35
|
+
options[:hash_key] ||= {key.to_sym => :string}
|
36
|
+
options[:range_key] = {options[:range_key].to_sym => :number} if options[:range_key]
|
37
|
+
table = @@connection.tables.create(table_name, 100, 20, options)
|
36
38
|
sleep 0.5 while table.status == :creating
|
37
39
|
return table
|
38
40
|
end
|
39
41
|
|
40
42
|
# DeleteItem
|
41
|
-
def delete_item(table_name, key)
|
43
|
+
def delete_item(table_name, key, range_key = nil)
|
42
44
|
table = @@connection.tables[table_name]
|
43
45
|
table.load_schema
|
44
|
-
result = table.
|
46
|
+
result = if table.composite_key?
|
47
|
+
table.items.at(key, range_key)
|
48
|
+
else
|
49
|
+
table.items[key]
|
50
|
+
end
|
45
51
|
result.delete unless result.attributes.to_h.empty?
|
46
52
|
true
|
47
53
|
end
|
@@ -54,15 +60,21 @@ module Dynamoid
|
|
54
60
|
# DescribeTable
|
55
61
|
|
56
62
|
# GetItem
|
57
|
-
def get_item(table_name, key)
|
63
|
+
def get_item(table_name, key, range_key = nil)
|
58
64
|
table = @@connection.tables[table_name]
|
59
65
|
table.load_schema
|
60
|
-
result = table.
|
66
|
+
result = if table.composite_key?
|
67
|
+
table.items.at(key, range_key)
|
68
|
+
else
|
69
|
+
table.items[key]
|
70
|
+
end.attributes.to_h
|
61
71
|
if result.empty?
|
62
72
|
nil
|
63
73
|
else
|
64
74
|
result.symbolize_keys!
|
65
75
|
end
|
76
|
+
rescue
|
77
|
+
raise [table_name, key, range_key].inspect
|
66
78
|
end
|
67
79
|
|
68
80
|
# ListTables
|
@@ -78,8 +90,17 @@ module Dynamoid
|
|
78
90
|
end
|
79
91
|
|
80
92
|
# Query
|
81
|
-
def query(table_name,
|
82
|
-
|
93
|
+
def query(table_name, opts = {})
|
94
|
+
table = @@connection.tables[table_name]
|
95
|
+
table.load_schema
|
96
|
+
|
97
|
+
if table.composite_key?
|
98
|
+
results = []
|
99
|
+
table.items.query(opts).each {|data| results << data.attributes.to_h.symbolize_keys!}
|
100
|
+
results
|
101
|
+
else
|
102
|
+
get_item(table_name, opts[:hash_value])
|
103
|
+
end
|
83
104
|
end
|
84
105
|
|
85
106
|
# Scan
|
@@ -17,21 +17,27 @@ module Dynamoid
|
|
17
17
|
Hash.new { |h, k| h[k] = Array.new }.tap do |hash|
|
18
18
|
options.each do |table_name, keys|
|
19
19
|
table = data[table_name]
|
20
|
-
|
21
|
-
|
20
|
+
if table[:range_key]
|
21
|
+
Array(keys).each do |hash_key, range_key|
|
22
|
+
hash[table_name] << get_item(table_name, hash_key, range_key)
|
23
|
+
end
|
24
|
+
else
|
25
|
+
Array(keys).each do |key|
|
26
|
+
hash[table_name] << get_item(table_name, key)
|
27
|
+
end
|
22
28
|
end
|
23
29
|
end
|
24
30
|
end
|
25
31
|
end
|
26
32
|
|
27
33
|
# CreateTable
|
28
|
-
def create_table(table_name, key)
|
29
|
-
data[table_name] = {:
|
34
|
+
def create_table(table_name, key, options = {})
|
35
|
+
data[table_name] = {:hash_key => key, :range_key => options[:range_key], :data => {}}
|
30
36
|
end
|
31
37
|
|
32
38
|
# DeleteItem
|
33
|
-
def delete_item(table_name, key)
|
34
|
-
data[table_name][:data].delete(key)
|
39
|
+
def delete_item(table_name, key, range_key = nil)
|
40
|
+
data[table_name][:data].delete("#{key}.#{range_key}")
|
35
41
|
end
|
36
42
|
|
37
43
|
# DeleteTable
|
@@ -42,8 +48,12 @@ module Dynamoid
|
|
42
48
|
# DescribeTable
|
43
49
|
|
44
50
|
# GetItem
|
45
|
-
def get_item(table_name, key)
|
46
|
-
data[table_name][:data]
|
51
|
+
def get_item(table_name, key, range_key = nil)
|
52
|
+
if data[table_name][:data]
|
53
|
+
data[table_name][:data]["#{key}.#{range_key}"]
|
54
|
+
else
|
55
|
+
nil
|
56
|
+
end
|
47
57
|
end
|
48
58
|
|
49
59
|
# ListTables
|
@@ -54,18 +64,33 @@ module Dynamoid
|
|
54
64
|
# PutItem
|
55
65
|
def put_item(table_name, object)
|
56
66
|
table = data[table_name]
|
57
|
-
table[:data][object[table[:
|
67
|
+
table[:data][object[table[:hash_key]]]
|
68
|
+
table[:data]["#{object[table[:hash_key]]}.#{object[table[:range_key]]}"] = object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)}
|
58
69
|
end
|
59
70
|
|
60
71
|
# Query
|
61
|
-
def query(table_name,
|
62
|
-
|
72
|
+
def query(table_name, opts = {})
|
73
|
+
id = opts[:hash_value]
|
74
|
+
range_key = data[table_name][:range_key]
|
75
|
+
if opts[:range_value]
|
76
|
+
data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && opts[:range_value].include?(v[range_key])}
|
77
|
+
elsif opts[:range_greater_than]
|
78
|
+
data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] > opts[:range_greater_than]}
|
79
|
+
elsif opts[:range_less_than]
|
80
|
+
data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] < opts[:range_less_than]}
|
81
|
+
elsif opts[:range_gte]
|
82
|
+
data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] >= opts[:range_gte]}
|
83
|
+
elsif opts[:range_lte]
|
84
|
+
data[table_name][:data].values.find_all{|v| v[:id] == id && !v[range_key].nil? && v[range_key] <= opts[:range_lte]}
|
85
|
+
else
|
86
|
+
get_item(table_name, id)
|
87
|
+
end
|
63
88
|
end
|
64
89
|
|
65
90
|
# Scan
|
66
91
|
def scan(table_name, scan_hash)
|
67
92
|
return [] if data[table_name].nil?
|
68
|
-
data[table_name][:data].values.select{|d| scan_hash.all?{|k, v| !d[k].nil? && d[k] == v}}
|
93
|
+
data[table_name][:data].values.flatten.select{|d| scan_hash.all?{|k, v| !d[k].nil? && d[k] == v}}
|
69
94
|
end
|
70
95
|
|
71
96
|
# UpdateItem
|
@@ -9,6 +9,14 @@ module Dynamoid #:nodoc:
|
|
9
9
|
def ==(other)
|
10
10
|
target == other
|
11
11
|
end
|
12
|
+
|
13
|
+
def method_missing(method, *args)
|
14
|
+
if target.respond_to?(method)
|
15
|
+
target.send(method, *args)
|
16
|
+
else
|
17
|
+
super
|
18
|
+
end
|
19
|
+
end
|
12
20
|
|
13
21
|
private
|
14
22
|
|
@@ -17,7 +25,7 @@ module Dynamoid #:nodoc:
|
|
17
25
|
end
|
18
26
|
|
19
27
|
def target_association
|
20
|
-
key_name = source.class.to_s.
|
28
|
+
key_name = source.class.to_s.singularize.downcase.to_sym
|
21
29
|
guess = target_class.associations[key_name]
|
22
30
|
return nil if guess.nil? || guess[:type] != :belongs_to
|
23
31
|
key_name
|
@@ -33,4 +41,4 @@ module Dynamoid #:nodoc:
|
|
33
41
|
end
|
34
42
|
end
|
35
43
|
|
36
|
-
end
|
44
|
+
end
|
data/lib/dynamoid/criteria.rb
CHANGED
@@ -33,16 +33,25 @@ module Dynamoid #:nodoc:
|
|
33
33
|
private
|
34
34
|
|
35
35
|
def records
|
36
|
-
return records_with_index
|
36
|
+
return records_with_index if index
|
37
37
|
records_without_index
|
38
38
|
end
|
39
39
|
|
40
40
|
def records_with_index
|
41
|
-
ids =
|
41
|
+
ids = if index.range_key?
|
42
|
+
Dynamoid::Adapter.query(index.table_name, index_query).collect{|r| r[:ids]}.inject(Set.new) {|set, result| set + result}
|
43
|
+
else
|
44
|
+
results = Dynamoid::Adapter.read(index.table_name, index_query[:hash_value])
|
45
|
+
if results
|
46
|
+
results[:ids]
|
47
|
+
else
|
48
|
+
[]
|
49
|
+
end
|
50
|
+
end
|
42
51
|
if ids.nil? || ids.empty?
|
43
52
|
[]
|
44
53
|
else
|
45
|
-
Array(source.find(ids
|
54
|
+
Array(source.find(ids.to_a))
|
46
55
|
end
|
47
56
|
end
|
48
57
|
|
@@ -54,12 +63,39 @@ module Dynamoid #:nodoc:
|
|
54
63
|
Dynamoid::Adapter.scan(source.table_name, query).collect {|hash| source.new(hash)}
|
55
64
|
end
|
56
65
|
|
57
|
-
def
|
58
|
-
|
66
|
+
def index_query
|
67
|
+
values = index.values(query)
|
68
|
+
{}.tap do |hash|
|
69
|
+
hash[:hash_value] = values[:hash_value]
|
70
|
+
if index.range_key?
|
71
|
+
key = query.keys.find{|k| k.to_s.include?('.')}
|
72
|
+
if key
|
73
|
+
if query[key].is_a?(Range)
|
74
|
+
hash[:range_value] = query[key]
|
75
|
+
else
|
76
|
+
val = query[key].to_f
|
77
|
+
case key.split('.').last
|
78
|
+
when 'gt'
|
79
|
+
hash[:range_greater_than] = val
|
80
|
+
when 'lt'
|
81
|
+
hash[:range_less_than] = val
|
82
|
+
when 'gte'
|
83
|
+
hash[:range_gte] = val
|
84
|
+
when 'lte'
|
85
|
+
hash[:range_lte] = val
|
86
|
+
end
|
87
|
+
end
|
88
|
+
else
|
89
|
+
raise Dynamoid::Errors::MissingRangeKey, 'This index requires a range key'
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
59
93
|
end
|
60
94
|
|
61
95
|
def index
|
62
|
-
|
96
|
+
index = source.find_index(query.keys.collect{|k| k.to_s.split('.').first})
|
97
|
+
return nil if index.blank?
|
98
|
+
index
|
63
99
|
end
|
64
100
|
end
|
65
101
|
|
data/lib/dynamoid/errors.rb
CHANGED
data/lib/dynamoid/indexes.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
+
require 'dynamoid/indexes/index'
|
3
|
+
|
2
4
|
module Dynamoid #:nodoc:
|
3
5
|
|
4
6
|
# Builds all indexes present on the model.
|
@@ -8,53 +10,39 @@ module Dynamoid #:nodoc:
|
|
8
10
|
included do
|
9
11
|
class_attribute :indexes
|
10
12
|
|
11
|
-
self.indexes =
|
13
|
+
self.indexes = {}
|
12
14
|
end
|
13
15
|
|
14
16
|
module ClassMethods
|
15
17
|
def index(name, options = {})
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
create_indexes
|
20
|
-
end
|
21
|
-
|
22
|
-
def create_indexes
|
23
|
-
self.indexes.each do |index|
|
24
|
-
self.create_table(index_table_name(index), :id) unless self.table_exists?(index_table_name(index))
|
25
|
-
end
|
18
|
+
index = Dynamoid::Indexes::Index.new(self, name, options)
|
19
|
+
self.indexes[index.name] = index
|
20
|
+
create_indexes
|
26
21
|
end
|
27
22
|
|
28
|
-
def
|
29
|
-
|
23
|
+
def find_index(index)
|
24
|
+
self.indexes[Array(index).collect(&:to_s).sort.collect(&:to_sym)]
|
30
25
|
end
|
31
26
|
|
32
|
-
def
|
33
|
-
|
27
|
+
def create_indexes
|
28
|
+
self.indexes.each do |name, index|
|
29
|
+
opts = index.range_key? ? {:range_key => :range} : {}
|
30
|
+
self.create_table(index.table_name, :id, opts) unless self.table_exists?(index.table_name)
|
31
|
+
end
|
34
32
|
end
|
35
33
|
end
|
36
34
|
|
37
|
-
def key_for_index(index)
|
38
|
-
self.class.key_for_index(index, index.collect{|i| self.send(i)})
|
39
|
-
end
|
40
|
-
|
41
35
|
def save_indexes
|
42
|
-
self.class.indexes.each do |index|
|
43
|
-
|
44
|
-
existing = Dynamoid::Adapter.read(self.class.index_table_name(index), self.key_for_index(index))
|
45
|
-
ids = existing ? existing[:ids] : Set.new
|
46
|
-
Dynamoid::Adapter.write(self.class.index_table_name(index), {:id => self.key_for_index(index), :ids => ids.merge([self.id])})
|
36
|
+
self.class.indexes.each do |name, index|
|
37
|
+
index.save(self)
|
47
38
|
end
|
48
39
|
end
|
49
40
|
|
50
41
|
def delete_indexes
|
51
|
-
self.class.indexes.each do |index|
|
52
|
-
|
53
|
-
existing = Dynamoid::Adapter.read(self.class.index_table_name(index), self.key_for_index(index))
|
54
|
-
next unless existing && existing[:ids]
|
55
|
-
Dynamoid::Adapter.write(self.class.index_table_name(index), {:id => self.key_for_index(index), :ids => existing[:ids] - [self.id]})
|
42
|
+
self.class.indexes.each do |name, index|
|
43
|
+
index.delete(self)
|
56
44
|
end
|
57
45
|
end
|
58
46
|
end
|
59
47
|
|
60
|
-
end
|
48
|
+
end
|