minidynamo 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/Gemfile +12 -0
- data/Gemfile.lock +39 -0
- data/LICENSE.txt +19 -0
- data/README.md +83 -0
- data/Rakefile +53 -0
- data/VERSION +1 -0
- data/lib/minidynamo.rb +3 -0
- data/lib/minidynamo/check_digit.rb +37 -0
- data/lib/minidynamo/model.rb +74 -0
- data/lib/minidynamo/model/definition_helpers.rb +41 -0
- data/lib/minidynamo/model/dynamo_db_overloads.rb +71 -0
- data/lib/minidynamo/model/finder_overloads.rb +74 -0
- data/lib/minidynamo/model/keys.rb +45 -0
- data/minidynamo.gemspec +66 -0
- data/model2.rb +199 -0
- data/test/helper.rb +18 -0
- data/test/test_minidynamo.rb +7 -0
- metadata +132 -0
data/.document
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
gem "aws-sdk", "~> 1.11.2"
|
4
|
+
|
5
|
+
# Add dependencies to develop your gem here.
|
6
|
+
# Include everything needed to run rake, tests, features, etc.
|
7
|
+
group :development do
|
8
|
+
gem "shoulda", ">= 0"
|
9
|
+
gem "rdoc", "~> 3.12"
|
10
|
+
gem "jeweler", "~> 1.8.4"
|
11
|
+
#gem "simplecov", "~> 0.7.1"
|
12
|
+
end
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
activesupport (3.2.13)
|
5
|
+
i18n (= 0.6.1)
|
6
|
+
multi_json (~> 1.0)
|
7
|
+
aws-sdk (1.11.2)
|
8
|
+
json (~> 1.4)
|
9
|
+
nokogiri (< 1.6.0)
|
10
|
+
uuidtools (~> 2.1)
|
11
|
+
git (1.2.5)
|
12
|
+
i18n (0.6.1)
|
13
|
+
jeweler (1.8.4)
|
14
|
+
bundler (~> 1.0)
|
15
|
+
git (>= 1.2.5)
|
16
|
+
rake
|
17
|
+
rdoc
|
18
|
+
json (1.8.0)
|
19
|
+
multi_json (1.7.6)
|
20
|
+
nokogiri (1.5.10)
|
21
|
+
rake (10.0.4)
|
22
|
+
rdoc (3.12.2)
|
23
|
+
json (~> 1.4)
|
24
|
+
shoulda (3.5.0)
|
25
|
+
shoulda-context (~> 1.0, >= 1.0.1)
|
26
|
+
shoulda-matchers (>= 1.4.1, < 3.0)
|
27
|
+
shoulda-context (1.1.2)
|
28
|
+
shoulda-matchers (2.2.0)
|
29
|
+
activesupport (>= 3.0.0)
|
30
|
+
uuidtools (2.1.4)
|
31
|
+
|
32
|
+
PLATFORMS
|
33
|
+
ruby
|
34
|
+
|
35
|
+
DEPENDENCIES
|
36
|
+
aws-sdk (~> 1.11.2)
|
37
|
+
jeweler (~> 1.8.4)
|
38
|
+
rdoc (~> 3.12)
|
39
|
+
shoulda
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
This gem contains modified code from Amazon's AWS SDK for Ruby.
|
2
|
+
The original license comes from http://aws.amazon.com/apache2.0/
|
3
|
+
|
4
|
+
Original copyright notice from the aws-sdk gem
|
5
|
+
==============================================
|
6
|
+
|
7
|
+
Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
8
|
+
|
9
|
+
Licensed under the Apache License, Version 2.0 (the "License"). You
|
10
|
+
may not use this file except in compliance with the License. A copy of
|
11
|
+
the License is located at
|
12
|
+
|
13
|
+
http://aws.amazon.com/apache2.0/
|
14
|
+
|
15
|
+
or in the "license" file accompanying this file. This file is
|
16
|
+
distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
17
|
+
ANY KIND, either express or implied. See the License for the specific
|
18
|
+
language governing permissions and limitations under the License.
|
19
|
+
|
data/README.md
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
Minidynamo
|
2
|
+
==============
|
3
|
+
|
4
|
+
Monkey patch for AWS ruby SDK HashModel to support models backed by DynamoDB tables with hash-range keys.
|
5
|
+
|
6
|
+
## What's minidynamo for?
|
7
|
+
|
8
|
+
|
9
|
+
As of June 2013 AWS-sdk for ruby HasModel for DynamoDB does not support working with tables having a hash-range main key. Additionally the main key in hash tables is alway called :id.
|
10
|
+
|
11
|
+
Another well known DynamoDB ORM is Dynamoid, but they lack this as well. Most likely because they've built on top of HashModel and hence they are subject to its current functionality.
|
12
|
+
|
13
|
+
Having the ability to perform queries in hash-range tables and the recent addition of secondary indexes for this kind of tables, we monkey patched the HashModel class to offer some initial support.
|
14
|
+
|
15
|
+
Depending on when and how Amazon supports this kind of tables, Minidynamo will continue to be supported. Since this gem is very small and mostly based on their own code, we're optimistic that Amazon will implement something similar. In the meantime, we hope this is useful.
|
16
|
+
|
17
|
+
Minidynamo offers the following compared to HashModel:
|
18
|
+
|
19
|
+
* Definition of tables and initial throughput with class methods á la Rails.
|
20
|
+
* Custom name(s) for the primary key column(s)
|
21
|
+
* For hash-range tables find() issues a rangeless query.
|
22
|
+
* find_by_#{hash_key}() is created by default when the main key hash column name isn't :id
|
23
|
+
* There's a create table method for each Model that accepts 0 parameteres and only uses the information given in the table definition.
|
24
|
+
|
25
|
+
This simple additions, even though still not offering the full range of posibilities for hash-range tables in DynamoDB current API, will cover lots of use cases. We currently use it in more than 3 internal applications. Hash-range + DynamoDB in this applications enable low latency data availability to improve response times in internal components and make scaling a matter of clicks.
|
26
|
+
|
27
|
+
## Current limitations
|
28
|
+
|
29
|
+
* There's currently no definition of extra indexes.
|
30
|
+
* Besides rangeless_query to find items in hash-range tables by only providing the hash value, there's no other querying functionality.
|
31
|
+
* Nothing has been done regarding scoping, scanning or anything related to find more than the rangeless_query method listed below.
|
32
|
+
|
33
|
+
## Instructions
|
34
|
+
|
35
|
+
Include minidynamo in your Gemfile.
|
36
|
+
|
37
|
+
```ruby
|
38
|
+
gem 'minidynamo', '~> 0.1.0'
|
39
|
+
```
|
40
|
+
|
41
|
+
Don't forget to do `bundle install` right after.
|
42
|
+
|
43
|
+
### Defining a table
|
44
|
+
|
45
|
+
```ruby
|
46
|
+
class TestModel < Minidynamo::Model
|
47
|
+
|
48
|
+
table name: "my_table",
|
49
|
+
hash_key: {:my_attribute => :string},
|
50
|
+
range_key: {:my_range_name => :string}
|
51
|
+
|
52
|
+
initial_througput read_capacity: 5, write_capacity: 5
|
53
|
+
|
54
|
+
timestamps
|
55
|
+
|
56
|
+
field :custom_field, :string
|
57
|
+
|
58
|
+
end
|
59
|
+
```
|
60
|
+
|
61
|
+
With `field` the valid types are those that HashModel offers in the form of `{type}_attr` methods. For example:
|
62
|
+
|
63
|
+
* string_attr, hence you can use :string as a type in field
|
64
|
+
* date_attr, hence you can use :date as a type in field
|
65
|
+
* integer_attr, hence you can use :integer as a type in field
|
66
|
+
|
67
|
+
`field` only calls those methods. This is pure synthactic sugar, but we think it resembles a lot better the way we've become used to see ruby syntax, specially if you come from the Rails world.
|
68
|
+
|
69
|
+
timestamps, as any other methods available via HashModel continue to be ready for you to use. You can check more in the [HasModel docs](http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html#!http%3A//docs.aws.amazon.com/AWSRubySDK/latest/AWS/DynamoDB.html).
|
70
|
+
|
71
|
+
You can still work with hash-only tables. Just omit the range_key part in the call to `table`.
|
72
|
+
|
73
|
+
You could go and create the table by calling `TestModel.create_table`
|
74
|
+
|
75
|
+
## Contributing to minidynamo
|
76
|
+
|
77
|
+
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
78
|
+
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
|
79
|
+
* Fork the project.
|
80
|
+
* Start a feature/bugfix branch.
|
81
|
+
* Commit and push until you are happy with your contribution.
|
82
|
+
* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
|
83
|
+
* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so we can cherry-pick around it.
|
data/Rakefile
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
begin
|
6
|
+
Bundler.setup(:default, :development)
|
7
|
+
rescue Bundler::BundlerError => e
|
8
|
+
$stderr.puts e.message
|
9
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
10
|
+
exit e.status_code
|
11
|
+
end
|
12
|
+
require 'rake'
|
13
|
+
|
14
|
+
require 'jeweler'
|
15
|
+
Jeweler::Tasks.new do |gem|
|
16
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
17
|
+
gem.name = "minidynamo"
|
18
|
+
gem.homepage = "http://github.com/vlipco/minidynamo"
|
19
|
+
gem.license = "MIT"
|
20
|
+
gem.summary = %Q{AWS SDK HashModel extension to support DynamoDB tables with hash-range keys}
|
21
|
+
gem.description = %Q{Monkey patch for AWS ruby SDK HashModel to support model backed by DynamoDB tables with hash-range keys. It also provides some convenience methods to define your model in a more expressive rails style.}
|
22
|
+
gem.email = "info@vlipco.co"
|
23
|
+
gem.authors = ["David Pelaez"]
|
24
|
+
# dependencies defined in Gemfile
|
25
|
+
end
|
26
|
+
Jeweler::RubygemsDotOrgTasks.new
|
27
|
+
|
28
|
+
require 'rake/testtask'
|
29
|
+
Rake::TestTask.new(:test) do |test|
|
30
|
+
test.libs << 'lib' << 'test'
|
31
|
+
test.pattern = 'test/**/test_*.rb'
|
32
|
+
test.verbose = true
|
33
|
+
end
|
34
|
+
|
35
|
+
#require 'rcov/rcovtask'
|
36
|
+
#Rcov::RcovTask.new do |test|
|
37
|
+
# test.libs << 'test'
|
38
|
+
# test.pattern = 'test/**/test_*.rb'
|
39
|
+
# test.verbose = true
|
40
|
+
# test.rcov_opts << '--exclude "gems/*"'
|
41
|
+
#end
|
42
|
+
|
43
|
+
task :default => :test
|
44
|
+
|
45
|
+
require 'rdoc/task'
|
46
|
+
Rake::RDocTask.new do |rdoc|
|
47
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
48
|
+
|
49
|
+
rdoc.rdoc_dir = 'rdoc'
|
50
|
+
rdoc.title = "minidynamo #{version}"
|
51
|
+
rdoc.rdoc_files.include('README*')
|
52
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
53
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/lib/minidynamo.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# Check digit generator & validator using UPC barcode algorithm
|
2
|
+
# as described in http://www.gs1.org/barcodes/support/check_digit_calculator
|
3
|
+
|
4
|
+
|
5
|
+
class CheckDigit
|
6
|
+
class << self
|
7
|
+
|
8
|
+
def calculate_check_digit(number)
|
9
|
+
(10 - (check_sum(number)%10)) % 10
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
|
14
|
+
def valid?(number)
|
15
|
+
digits = to_digits number
|
16
|
+
check = digits.pop
|
17
|
+
check == calculate_check_digit(digits)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def check_sum(number)
|
23
|
+
digits = to_digits number
|
24
|
+
values = digits.each_slice(2).map do |x, y|
|
25
|
+
y ||= 0
|
26
|
+
[x*3, y]
|
27
|
+
end
|
28
|
+
values.flatten.inject(:+)
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def to_digits(number)
|
33
|
+
number.to_s.chars.map &:to_i
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# Based on AWS::Record::HashModel from aws-sdk gem.
|
2
|
+
# Check the license file included in this gem or the original version at:
|
3
|
+
# http://aws.amazon.com/apache2.0/
|
4
|
+
|
5
|
+
module Minidynamo
|
6
|
+
class Model < AWS::Record::HashModel
|
7
|
+
|
8
|
+
require 'minidynamo/model/keys'
|
9
|
+
require 'minidynamo/model/definition_helpers'
|
10
|
+
require 'minidynamo/model/dynamo_db_overloads'
|
11
|
+
require 'minidynamo/model/finder_overloads'
|
12
|
+
|
13
|
+
#require 'aws/record/hash_model/attributes'
|
14
|
+
#require 'aws/record/hash_model/finder_methods'
|
15
|
+
#require 'aws/record/hash_model/scope'
|
16
|
+
|
17
|
+
#extend AWS::Record::AbstractBase
|
18
|
+
|
19
|
+
extend Keys
|
20
|
+
extend DefinitionHelpers
|
21
|
+
extend DynamoDBOverloads
|
22
|
+
extend FinderOverloads
|
23
|
+
|
24
|
+
|
25
|
+
|
26
|
+
class << self
|
27
|
+
|
28
|
+
#
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
#private
|
33
|
+
#def create_storage
|
34
|
+
# attributes = serialize_attributes.merge('id' => @_id)
|
35
|
+
# dynamo_db_table.items.create(attributes, opt_lock_conditions)
|
36
|
+
#end
|
37
|
+
#
|
38
|
+
#private
|
39
|
+
#def update_storage
|
40
|
+
# # Only enumerating dirty (i.e. changed) attributes. Empty
|
41
|
+
# # (nil and empty set) values are deleted, the others are replaced.
|
42
|
+
# dynamo_db_item.attributes.update(opt_lock_conditions) do |u|
|
43
|
+
# changed.each do |attr_name|
|
44
|
+
# attribute = self.class.attribute_for(attr_name)
|
45
|
+
# value = serialize_attribute(attribute, @_data[attr_name])
|
46
|
+
# if value.nil? or value == []
|
47
|
+
# u.delete(attr_name)
|
48
|
+
# else
|
49
|
+
# u.set(attr_name => value)
|
50
|
+
# end
|
51
|
+
# end
|
52
|
+
# end
|
53
|
+
#end
|
54
|
+
#
|
55
|
+
#private
|
56
|
+
#def delete_storage
|
57
|
+
# dynamo_db_item.delete(opt_lock_conditions)
|
58
|
+
#end
|
59
|
+
#
|
60
|
+
#private
|
61
|
+
#def deserialize_item_data data
|
62
|
+
# data.inject({}) do |hash,(attr_name,value)|
|
63
|
+
# if attribute = self.class.attributes[attr_name]
|
64
|
+
# hash[attr_name] = value.is_a?(Set) ?
|
65
|
+
# value.map{|v| attribute.deserialize(v) } :
|
66
|
+
# attribute.deserialize(value)
|
67
|
+
# end
|
68
|
+
# hash
|
69
|
+
# end
|
70
|
+
#end
|
71
|
+
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Minidynamo
|
2
|
+
class Model
|
3
|
+
module DefinitionHelpers
|
4
|
+
|
5
|
+
# For all the fields with defaults
|
6
|
+
attr_writer :write_capacity, :read_capacity
|
7
|
+
|
8
|
+
attr_accessor :range_key
|
9
|
+
|
10
|
+
def table options = {}
|
11
|
+
set_shard_name options[:name]
|
12
|
+
self.hash_key = options[:hash_key] unless options[:hash_key].nil?
|
13
|
+
self.range_key = options[:range_key] unless options[:range_key].nil?
|
14
|
+
end
|
15
|
+
|
16
|
+
#
|
17
|
+
# TABLE STRUCTURE HELPERS
|
18
|
+
#
|
19
|
+
|
20
|
+
def field key, type, options = {}
|
21
|
+
method_name = "#{type.to_s}_attr".to_sym
|
22
|
+
puts "CALLING #{method_name}"
|
23
|
+
send method_name, key, options
|
24
|
+
end
|
25
|
+
|
26
|
+
def initial_througput options = {}
|
27
|
+
self.read_capacity = options[:read_capacity]
|
28
|
+
self.write_capacity = options[:write_capacity]
|
29
|
+
end
|
30
|
+
|
31
|
+
def read_capacity
|
32
|
+
@read_capacity || 10
|
33
|
+
end
|
34
|
+
|
35
|
+
def write_capacity
|
36
|
+
@write_capacity || 10
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module Minidynamo
|
2
|
+
class Model
|
3
|
+
module DynamoDBOverloads
|
4
|
+
|
5
|
+
def self.extended base
|
6
|
+
base.send(:extend, ClassMethods)
|
7
|
+
base.send(:include, InstanceMethods)
|
8
|
+
end
|
9
|
+
|
10
|
+
module ClassMethods
|
11
|
+
|
12
|
+
# Changed: Don't accept parameters when creating, use the declared options
|
13
|
+
# inside the model
|
14
|
+
def create_table
|
15
|
+
create_opts = {}
|
16
|
+
create_opts[:hash_key] = hash_key
|
17
|
+
create_opts[:range_key] = range_key if range_key
|
18
|
+
|
19
|
+
dynamo_db.tables.create dynamo_db_table_name,
|
20
|
+
read_capacity,
|
21
|
+
write_capacity,
|
22
|
+
create_opts
|
23
|
+
end
|
24
|
+
|
25
|
+
# @return [DynamoDB::Table]
|
26
|
+
# changed to use hash key other than id
|
27
|
+
# @api private
|
28
|
+
def dynamo_db_table shard_name = nil
|
29
|
+
table = dynamo_db.tables[dynamo_db_table_name(shard_name)]
|
30
|
+
table.hash_key = hash_key #[:id, :string]
|
31
|
+
table.range_key = range_key if range_key
|
32
|
+
|
33
|
+
#table.hash_key = {:public_token => :string }
|
34
|
+
#table.range_key = {:created_at => :string }
|
35
|
+
|
36
|
+
#table.hash_key = [:public_token, :string]
|
37
|
+
#table.range_key = [:created_at, :string]
|
38
|
+
table
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
module InstanceMethods
|
45
|
+
|
46
|
+
def serialize_current attr_name
|
47
|
+
serialized_value = attributes[attr_name]
|
48
|
+
attr_object = self.class.attribute_for(attr_name)
|
49
|
+
serialize_attribute attr_object, serialized_value
|
50
|
+
end
|
51
|
+
|
52
|
+
# @return [DynamoDB::Item] Returns a reference to the item as stored in
|
53
|
+
# simple db.
|
54
|
+
# obtain items ALSO if there's a range key
|
55
|
+
# @api private
|
56
|
+
private
|
57
|
+
def dynamo_db_item
|
58
|
+
hash_value = serialize_current self.class.hash_key_attribute_name
|
59
|
+
if self.class.range_key
|
60
|
+
range_value = serialize_current self.class.range_key_attribute_name
|
61
|
+
dynamo_db_table.items[hash_value, range_value]
|
62
|
+
else
|
63
|
+
dynamo_db_table.items[hash_value]
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Minidynamo
|
2
|
+
class Model
|
3
|
+
module FinderOverloads
|
4
|
+
|
5
|
+
def find_by_id id, options = {}
|
6
|
+
|
7
|
+
table = dynamo_db_table(options[:shard])
|
8
|
+
|
9
|
+
data = table.items[id].attributes.to_h
|
10
|
+
|
11
|
+
raise RecordNotFound, "no data found for id: #{id}" if data.empty?
|
12
|
+
|
13
|
+
obj = self.new(:shard => table)
|
14
|
+
obj.send(:hydrate, id, data)
|
15
|
+
obj
|
16
|
+
end
|
17
|
+
|
18
|
+
alias_method :[], :find_by_id
|
19
|
+
|
20
|
+
#def find *args
|
21
|
+
# new_scope.find(*args)
|
22
|
+
#end
|
23
|
+
|
24
|
+
def find *args
|
25
|
+
# If find(id), determine wether a query or regular find will work
|
26
|
+
# convert the first argument to integer, if it's a string it'll yield 0
|
27
|
+
|
28
|
+
# if the first argument is an integer and this is a table with range as main key
|
29
|
+
if args.length == 1 && !args[0].is_a?(Symbol) && hash_range_table?
|
30
|
+
items = rangeless_query args[0]
|
31
|
+
case items.length
|
32
|
+
when 0
|
33
|
+
raise AWS::Record::RecordNotFound, "no data found for #{hash_key_attribute}: #{args[0]}"
|
34
|
+
when 1
|
35
|
+
return items[0]
|
36
|
+
else
|
37
|
+
return items
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
#in any other case fall back to the default HashModel
|
42
|
+
new_scope.find(*args)
|
43
|
+
end
|
44
|
+
|
45
|
+
def rangeless_query(*args)
|
46
|
+
hkv = args[0]
|
47
|
+
result = dynamo_db.client.query :table_name => dynamo_db_table_name,
|
48
|
+
:consistent_read => true,
|
49
|
+
:hash_key_value => {hash_key_type => hkv}
|
50
|
+
|
51
|
+
# Convert the result to items
|
52
|
+
hashed_items = result["Items"]
|
53
|
+
items = []
|
54
|
+
|
55
|
+
hashed_items.each do |i|
|
56
|
+
i_data = {}
|
57
|
+
i.each do |k,v|
|
58
|
+
# Obtain the first key for the hash describing the value of a DDB column
|
59
|
+
k_data_type = i[k].keys[0]
|
60
|
+
i_data[k] = i[k][k_data_type]
|
61
|
+
end
|
62
|
+
#puts "SEL CLASS #{self.class}"
|
63
|
+
|
64
|
+
pt = new
|
65
|
+
pt.send :hydrate, i_data["id"], i_data
|
66
|
+
items << pt
|
67
|
+
end
|
68
|
+
|
69
|
+
items
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Minidynamo
|
2
|
+
class Model
|
3
|
+
module Keys
|
4
|
+
|
5
|
+
# For all the fields with defaults
|
6
|
+
attr_writer :write_capacity, :read_capacity
|
7
|
+
|
8
|
+
attr_accessor :range_key
|
9
|
+
|
10
|
+
def hash_key
|
11
|
+
@hash_key || {:id => :string}
|
12
|
+
end
|
13
|
+
|
14
|
+
def hash_key=(key)
|
15
|
+
@hash_key = key
|
16
|
+
hk = key.keys[0]
|
17
|
+
finder_method_name = "find_by_#{hk}".to_sym
|
18
|
+
self.define_singleton_method finder_method_name do |x|
|
19
|
+
find_by_id x
|
20
|
+
end
|
21
|
+
type = key[hk]
|
22
|
+
attribute_creator_method_name = "#{type.to_s}_attr".to_sym
|
23
|
+
|
24
|
+
send attribute_creator_method_name, hk unless attribute_creator_method_name == :find_by_id
|
25
|
+
end
|
26
|
+
|
27
|
+
def hash_key_attribute_name
|
28
|
+
hash_key.keys[0]
|
29
|
+
end
|
30
|
+
|
31
|
+
def range_key_attribute_name
|
32
|
+
range_key.keys[0]
|
33
|
+
end
|
34
|
+
|
35
|
+
def hash_key_type
|
36
|
+
dynamo_db_table.hash_key.type.to_s.chars.first.to_sym
|
37
|
+
end
|
38
|
+
|
39
|
+
def hash_range_table?
|
40
|
+
! range_key.nil?
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/minidynamo.gemspec
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "minidynamo"
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["David Pelaez"]
|
12
|
+
s.date = "2013-06-14"
|
13
|
+
s.description = "Monkey patch for AWS ruby SDK HashModel to support model backed by DynamoDB tables with hash-range keys. It also provides some convenience methods to define your model in a more expressive rails style."
|
14
|
+
s.email = "info@vlipco.co"
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE.txt",
|
17
|
+
"README.md"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".document",
|
21
|
+
"Gemfile",
|
22
|
+
"Gemfile.lock",
|
23
|
+
"LICENSE.txt",
|
24
|
+
"README.md",
|
25
|
+
"Rakefile",
|
26
|
+
"VERSION",
|
27
|
+
"lib/minidynamo.rb",
|
28
|
+
"lib/minidynamo/check_digit.rb",
|
29
|
+
"lib/minidynamo/model.rb",
|
30
|
+
"lib/minidynamo/model/definition_helpers.rb",
|
31
|
+
"lib/minidynamo/model/dynamo_db_overloads.rb",
|
32
|
+
"lib/minidynamo/model/finder_overloads.rb",
|
33
|
+
"lib/minidynamo/model/keys.rb",
|
34
|
+
"minidynamo.gemspec",
|
35
|
+
"model2.rb",
|
36
|
+
"test/helper.rb",
|
37
|
+
"test/test_minidynamo.rb"
|
38
|
+
]
|
39
|
+
s.homepage = "http://github.com/vlipco/minidynamo"
|
40
|
+
s.licenses = ["MIT"]
|
41
|
+
s.require_paths = ["lib"]
|
42
|
+
s.rubygems_version = "1.8.23"
|
43
|
+
s.summary = "AWS SDK HashModel extension to support DynamoDB tables with hash-range keys"
|
44
|
+
|
45
|
+
if s.respond_to? :specification_version then
|
46
|
+
s.specification_version = 3
|
47
|
+
|
48
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
49
|
+
s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.11.2"])
|
50
|
+
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
51
|
+
s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
|
52
|
+
s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
|
53
|
+
else
|
54
|
+
s.add_dependency(%q<aws-sdk>, ["~> 1.11.2"])
|
55
|
+
s.add_dependency(%q<shoulda>, [">= 0"])
|
56
|
+
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
57
|
+
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
58
|
+
end
|
59
|
+
else
|
60
|
+
s.add_dependency(%q<aws-sdk>, ["~> 1.11.2"])
|
61
|
+
s.add_dependency(%q<shoulda>, [">= 0"])
|
62
|
+
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
63
|
+
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
data/model2.rb
ADDED
@@ -0,0 +1,199 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
def precede_with_check(x)
|
4
|
+
check_digit = CheckDigit.calculate_check_digit(x)
|
5
|
+
"#{check_digit}#{x}"
|
6
|
+
end
|
7
|
+
|
8
|
+
|
9
|
+
module Minidynamo
|
10
|
+
|
11
|
+
|
12
|
+
# Wrap some HashModel behaviour and offers minor overrides for readability
|
13
|
+
# and to create tables using ranges.
|
14
|
+
class Model < AWS::Record::HashModel
|
15
|
+
|
16
|
+
class << self
|
17
|
+
|
18
|
+
# For all the fields with defaults
|
19
|
+
attr_writer :write_capacity, :read_capacity
|
20
|
+
|
21
|
+
attr_accessor :range_key
|
22
|
+
|
23
|
+
# Use the client table creator instead of the HashModel inheritance
|
24
|
+
def create_table
|
25
|
+
# TODO verify it's not already created
|
26
|
+
|
27
|
+
create_opts = {}
|
28
|
+
create_opts[:hash_key] = hash_key
|
29
|
+
create_opts[:range_key] = range_key if range_key
|
30
|
+
|
31
|
+
|
32
|
+
dynamo_db.tables.create dynamo_db_table_name,
|
33
|
+
read_capacity,
|
34
|
+
write_capacity,
|
35
|
+
create_opts
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
# Obtain the DynamoDB status of a table
|
40
|
+
def status
|
41
|
+
end
|
42
|
+
|
43
|
+
def count_total
|
44
|
+
|
45
|
+
initial_q = dynamo_db.client.scan :table_name => dynamo_db_table_name,
|
46
|
+
:count => true
|
47
|
+
count = initial_q["Count"]
|
48
|
+
if initial_q["LastEvaluatedKey"]
|
49
|
+
last = initial_q["LastEvaluatedKey"]
|
50
|
+
begin
|
51
|
+
q = dynamo_db.client.scan :table_name => dynamo_db_table_name,
|
52
|
+
:count => true,
|
53
|
+
:exclusive_start_key => last
|
54
|
+
count += q["Count"]
|
55
|
+
last = q["LastEvaluatedKey"]
|
56
|
+
end while last
|
57
|
+
end
|
58
|
+
count
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
def ddb_client
|
63
|
+
@client ||= AWS::DynamoDB.new.client
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
|
68
|
+
#
|
69
|
+
# TABLE THROUGHPUT HELPERS
|
70
|
+
#
|
71
|
+
|
72
|
+
def initial_througput options = {}
|
73
|
+
self.read_capacity = options[:read_capacity]
|
74
|
+
self.write_capacity = options[:write_capacity]
|
75
|
+
end
|
76
|
+
|
77
|
+
def read_capacity
|
78
|
+
@read_capacity || 10
|
79
|
+
end
|
80
|
+
|
81
|
+
def write_capacity
|
82
|
+
@write_capacity || 10
|
83
|
+
end
|
84
|
+
|
85
|
+
def hash_key
|
86
|
+
@hash_key || {:id => :string}
|
87
|
+
end
|
88
|
+
|
89
|
+
def table options = {}
|
90
|
+
set_shard_name options[:name]
|
91
|
+
self.hash_key = options[:hash_key] unless options[:hash_key].nil?
|
92
|
+
self.range_key = options[:range_key] unless options[:range_key].nil?
|
93
|
+
end
|
94
|
+
|
95
|
+
#
|
96
|
+
# TABLE STRUCTURE HELPERS
|
97
|
+
#
|
98
|
+
|
99
|
+
def field key, type, options = {}
|
100
|
+
method_name = "#{type.to_s}_attr".to_sym
|
101
|
+
puts "CALLING #{method_name}"
|
102
|
+
send method_name, key, options
|
103
|
+
end
|
104
|
+
|
105
|
+
def hash_key=(key)
|
106
|
+
@hash_key = key
|
107
|
+
hk = key.keys[0]
|
108
|
+
finder_method_name = "find_by_#{hk}".to_sym
|
109
|
+
self.define_singleton_method finder_method_name do |x|
|
110
|
+
find_by_id x
|
111
|
+
end
|
112
|
+
type = key[hk]
|
113
|
+
attribute_creator_method_name = "#{type.to_s}_attr".to_sym
|
114
|
+
puts "CREATING HASH KEY ATTR #{attribute_creator_method_name}"
|
115
|
+
send attribute_creator_method_name, hk
|
116
|
+
end
|
117
|
+
|
118
|
+
# OVERLOAD THE FINDER TO USE CUSTOM HASK KEYS IN ERRORS
|
119
|
+
|
120
|
+
alias_method :_find_by_id, :find_by_id
|
121
|
+
alias_method :_find, :find
|
122
|
+
|
123
|
+
#data = table.items[id].attributes.to_h
|
124
|
+
#
|
125
|
+
#raise RecordNotFound, "no data found for id: #{id}" if data.empty?
|
126
|
+
#
|
127
|
+
#obj = self.new(:shard => table)
|
128
|
+
#obj.send(:hydrate, id, data)
|
129
|
+
#obj
|
130
|
+
#
|
131
|
+
#.query :table_name => "sample-table", :consistent_read => true, :hash_key_value => {:s => "123"}
|
132
|
+
|
133
|
+
def find *args
|
134
|
+
# If find(id), determine wether a query or regular find will work
|
135
|
+
# convert the first argument to integer, if it's a string it'll yield 0
|
136
|
+
|
137
|
+
# if the first argument is an integer and this is a table with range as main key
|
138
|
+
if args.length == 1 && !args[0].is_a?(Symbol) && hash_range_table?
|
139
|
+
items = rangeless_query args[0]
|
140
|
+
case items.length
|
141
|
+
when 0
|
142
|
+
raise AWS::Record::RecordNotFound, "no data found for #{hash_key.keys[0]}: #{args[0]}"
|
143
|
+
when 1
|
144
|
+
return items[0]
|
145
|
+
else
|
146
|
+
return items
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
#in any other case fall back to the default HashModel
|
151
|
+
new_scope.find(*args)
|
152
|
+
end
|
153
|
+
|
154
|
+
# converts string > s, numeric > n
|
155
|
+
def hash_key_type
|
156
|
+
dynamo_db_table.hash_key.type.to_s.chars.first.to_sym
|
157
|
+
end
|
158
|
+
|
159
|
+
def rangeless_query(*args)
|
160
|
+
# Map to string if that's the case. For any other type of hash_key
|
161
|
+
# you will have to provide the converted value or errors might appear
|
162
|
+
hash_key_type == :s ? hkv = args[0].to_s : hkv = args[0]
|
163
|
+
result = dynamo_db.client.query :table_name => dynamo_db_table_name,
|
164
|
+
:consistent_read => true,
|
165
|
+
:hash_key_value => {hash_key_type => hkv}
|
166
|
+
|
167
|
+
# Convert the result to items
|
168
|
+
hashed_items = result["Items"]
|
169
|
+
items = []
|
170
|
+
hashed_items.each do |i|
|
171
|
+
i_data = {}
|
172
|
+
i.each do |k,v|
|
173
|
+
# Obtain the first key for the hash describing the value of a DDB column
|
174
|
+
k_data_type = i[k].keys[0]
|
175
|
+
i_data[k] = i[k][k_data_type]
|
176
|
+
end
|
177
|
+
pt = PublicToken.new
|
178
|
+
pt.send :hydrate, i_data["id"], i_data
|
179
|
+
items << pt
|
180
|
+
end
|
181
|
+
items
|
182
|
+
end
|
183
|
+
|
184
|
+
def hash_range_table?
|
185
|
+
! hash_key.nil?
|
186
|
+
end
|
187
|
+
|
188
|
+
def find_by_id *args
|
189
|
+
begin
|
190
|
+
hash_range_table? ? rangeless_query(args) : _find_by_id(args)
|
191
|
+
|
192
|
+
rescue AWS::Record::RecordNotFound
|
193
|
+
raise AWS::Record::RecordNotFound, "no data found for #{hash_key}: #{id}"
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
require 'shoulda'
|
12
|
+
|
13
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
14
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
15
|
+
require 'minidynamo'
|
16
|
+
|
17
|
+
class Test::Unit::TestCase
|
18
|
+
end
|
metadata
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: minidynamo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- David Pelaez
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-06-14 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: aws-sdk
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 1.11.2
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.11.2
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: shoulda
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: rdoc
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '3.12'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.12'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: jeweler
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ~>
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.8.4
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ~>
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.8.4
|
78
|
+
description: Monkey patch for AWS ruby SDK HashModel to support model backed by DynamoDB
|
79
|
+
tables with hash-range keys. It also provides some convenience methods to define
|
80
|
+
your model in a more expressive rails style.
|
81
|
+
email: info@vlipco.co
|
82
|
+
executables: []
|
83
|
+
extensions: []
|
84
|
+
extra_rdoc_files:
|
85
|
+
- LICENSE.txt
|
86
|
+
- README.md
|
87
|
+
files:
|
88
|
+
- .document
|
89
|
+
- Gemfile
|
90
|
+
- Gemfile.lock
|
91
|
+
- LICENSE.txt
|
92
|
+
- README.md
|
93
|
+
- Rakefile
|
94
|
+
- VERSION
|
95
|
+
- lib/minidynamo.rb
|
96
|
+
- lib/minidynamo/check_digit.rb
|
97
|
+
- lib/minidynamo/model.rb
|
98
|
+
- lib/minidynamo/model/definition_helpers.rb
|
99
|
+
- lib/minidynamo/model/dynamo_db_overloads.rb
|
100
|
+
- lib/minidynamo/model/finder_overloads.rb
|
101
|
+
- lib/minidynamo/model/keys.rb
|
102
|
+
- minidynamo.gemspec
|
103
|
+
- model2.rb
|
104
|
+
- test/helper.rb
|
105
|
+
- test/test_minidynamo.rb
|
106
|
+
homepage: http://github.com/vlipco/minidynamo
|
107
|
+
licenses:
|
108
|
+
- MIT
|
109
|
+
post_install_message:
|
110
|
+
rdoc_options: []
|
111
|
+
require_paths:
|
112
|
+
- lib
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
+
none: false
|
115
|
+
requirements:
|
116
|
+
- - ! '>='
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
none: false
|
121
|
+
requirements:
|
122
|
+
- - ! '>='
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
requirements: []
|
126
|
+
rubyforge_project:
|
127
|
+
rubygems_version: 1.8.23
|
128
|
+
signing_key:
|
129
|
+
specification_version: 3
|
130
|
+
summary: AWS SDK HashModel extension to support DynamoDB tables with hash-range keys
|
131
|
+
test_files: []
|
132
|
+
has_rdoc:
|