dynomite 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +3 -0
- data/CHANGELOG.md +28 -0
- data/Gemfile +6 -0
- data/README.md +141 -0
- data/Rakefile +2 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/docs/migrations/long-example.rb +123 -0
- data/docs/migrations/short-example.rb +36 -0
- data/dynomite.gemspec +29 -0
- data/lib/dynomite.rb +23 -0
- data/lib/dynomite/core.rb +25 -0
- data/lib/dynomite/db_config.rb +101 -0
- data/lib/dynomite/erb.rb +53 -0
- data/lib/dynomite/item.rb +291 -0
- data/lib/dynomite/log.rb +15 -0
- data/lib/dynomite/migration.rb +27 -0
- data/lib/dynomite/migration/common.rb +86 -0
- data/lib/dynomite/migration/dsl.rb +172 -0
- data/lib/dynomite/migration/dsl/base_secondary_index.rb +72 -0
- data/lib/dynomite/migration/dsl/global_secondary_index.rb +4 -0
- data/lib/dynomite/migration/dsl/local_secondary_index.rb +8 -0
- data/lib/dynomite/migration/executor.rb +30 -0
- data/lib/dynomite/migration/generator.rb +68 -0
- data/lib/dynomite/migration/templates/create_table.rb +32 -0
- data/lib/dynomite/migration/templates/update_table.rb +26 -0
- data/lib/dynomite/version.rb +3 -0
- metadata +141 -0
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Dynomite::Core
|
4
|
+
# Ensures trailing slash
|
5
|
+
# Useful for appending a './' in front of a path or leaving it alone.
|
6
|
+
# Returns: '/path/with/trailing/slash/' or './'
|
7
|
+
@@app_root = nil
|
8
|
+
def app_root
|
9
|
+
return @@app_root if @@app_root
|
10
|
+
@@app_root = ENV['APP_ROOT'] || ENV['JETS_ROOT'] || ENV['RAILS_ROOT']
|
11
|
+
@@app_root = '.' if @@app_root.nil? || @app_root == ''
|
12
|
+
@@app_root = "#{@@app_root}/" unless @@app_root.ends_with?('/')
|
13
|
+
@@app_root
|
14
|
+
end
|
15
|
+
|
16
|
+
@@logger = nil
|
17
|
+
def logger
|
18
|
+
return @@logger if @@logger
|
19
|
+
@@logger = Logger.new($stderr)
|
20
|
+
end
|
21
|
+
|
22
|
+
def logger=(value)
|
23
|
+
@@logger = value
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require "aws-sdk-dynamodb"
|
2
|
+
require 'fileutils'
|
3
|
+
require 'erb'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
module Dynomite::DbConfig
|
7
|
+
def self.included(base)
|
8
|
+
base.extend(ClassMethods)
|
9
|
+
end
|
10
|
+
|
11
|
+
def db
|
12
|
+
self.class.db
|
13
|
+
end
|
14
|
+
|
15
|
+
# NOTE: Class including Dynomite::DbConfig is required to have table_name method defined
|
16
|
+
def namespaced_table_name
|
17
|
+
[self.class.table_namespace, table_name].reject {|s| s.nil? || s.empty?}.join('-')
|
18
|
+
end
|
19
|
+
|
20
|
+
module ClassMethods
|
21
|
+
@@db = nil
|
22
|
+
def db
|
23
|
+
return @@db if @@db
|
24
|
+
|
25
|
+
config = db_config
|
26
|
+
endpoint = ENV['DYNAMODB_ENDPOINT'] || config['endpoint']
|
27
|
+
check_dynamodb_local!(endpoint)
|
28
|
+
|
29
|
+
Aws.config.update(endpoint: endpoint) if endpoint
|
30
|
+
@@db ||= Aws::DynamoDB::Client.new
|
31
|
+
end
|
32
|
+
|
33
|
+
# When endoint has been configured to point at dynamodb local: localhost:8000
|
34
|
+
# check if port 8000 is listening and timeout quickly. Or else it takes a
|
35
|
+
# for DynamoDB local to time out, about 10 seconds...
|
36
|
+
# This wastes less of the users time.
|
37
|
+
def check_dynamodb_local!(endpoint)
|
38
|
+
return unless endpoint && endpoint.include?("8000")
|
39
|
+
|
40
|
+
open = port_open?("127.0.0.1", 8000, 0.2)
|
41
|
+
unless open
|
42
|
+
raise "You have configured your app to use DynamoDB local, but it is not running. Please start DynamoDB local. Example: brew cask install dynamodb-local && dynamodb-local"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# Thanks: https://gist.github.com/ashrithr/5305786
|
47
|
+
def port_open?(ip, port, seconds=1)
|
48
|
+
# => checks if a port is open or not
|
49
|
+
Timeout::timeout(seconds) do
|
50
|
+
begin
|
51
|
+
TCPSocket.new(ip, port).close
|
52
|
+
true
|
53
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError
|
54
|
+
false
|
55
|
+
end
|
56
|
+
end
|
57
|
+
rescue Timeout::Error
|
58
|
+
false
|
59
|
+
end
|
60
|
+
|
61
|
+
# useful for specs
|
62
|
+
def db=(db)
|
63
|
+
@@db = db
|
64
|
+
end
|
65
|
+
|
66
|
+
def db_config
|
67
|
+
return @db_config if @db_config
|
68
|
+
|
69
|
+
if defined?(Jets)
|
70
|
+
config_path = "#{Jets.root}config/dynamodb.yml"
|
71
|
+
env = Jets.env
|
72
|
+
else
|
73
|
+
config_path = ENV['DYNAMODB_MODEL_CONFIG'] || "./config/dynamodb.yml"
|
74
|
+
env = ENV['DYNAMODB_MODEL_ENV'] || "development"
|
75
|
+
end
|
76
|
+
|
77
|
+
config = YAML.load(Dynomite::Erb.result(config_path))
|
78
|
+
@db_config ||= config[env] || {}
|
79
|
+
end
|
80
|
+
|
81
|
+
def table_namespace(*args)
|
82
|
+
case args.size
|
83
|
+
when 0
|
84
|
+
get_table_namespace
|
85
|
+
when 1
|
86
|
+
set_table_namespace(args[0])
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def get_table_namespace
|
91
|
+
return @table_namespace if defined?(@table_namespace)
|
92
|
+
|
93
|
+
config = db_config
|
94
|
+
@table_namespace = config['table_namespace']
|
95
|
+
end
|
96
|
+
|
97
|
+
def set_table_namespace(value)
|
98
|
+
@table_namespace = value
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
data/lib/dynomite/erb.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'erb'
|
2
|
+
|
3
|
+
# Renders Erb and provide better backtrace where there's an error
|
4
|
+
#
|
5
|
+
# Usage:
|
6
|
+
#
|
7
|
+
# result = Dynomite::Erb.result(path, key1: "val1", key2: "val2")
|
8
|
+
#
|
9
|
+
class Dynomite::Erb
|
10
|
+
include Dynomite::Log
|
11
|
+
|
12
|
+
class << self
|
13
|
+
def result(path, variables={})
|
14
|
+
set_template_variables(variables)
|
15
|
+
template = IO.read(path)
|
16
|
+
begin
|
17
|
+
ERB.new(template, nil, "-").result(binding)
|
18
|
+
rescue Exception => e
|
19
|
+
log(e)
|
20
|
+
log(e.backtrace) if ENV['DEBUG']
|
21
|
+
|
22
|
+
# how to know where ERB stopped? - https://www.ruby-forum.com/topic/182051
|
23
|
+
# syntax errors have the (erb):xxx info in e.message
|
24
|
+
# undefined variables have (erb):xxx info in e.backtrac
|
25
|
+
error_info = e.message.split("\n").grep(/\(erb\)/)[0]
|
26
|
+
error_info ||= e.backtrace.grep(/\(erb\)/)[0]
|
27
|
+
raise unless error_info # unable to find the (erb):xxx: error line
|
28
|
+
line = error_info.split(':')[1].to_i
|
29
|
+
log "Error evaluating ERB template on line #{line.to_s.colorize(:red)} of: #{path.sub(/^\.\//, '').colorize(:green)}"
|
30
|
+
|
31
|
+
template_lines = template.split("\n")
|
32
|
+
context = 5 # lines of context
|
33
|
+
top, bottom = [line-context-1, 0].max, line+context-1
|
34
|
+
spacing = template_lines.size.to_s.size
|
35
|
+
template_lines[top..bottom].each_with_index do |line_content, index|
|
36
|
+
line_number = top+index+1
|
37
|
+
if line_number == line
|
38
|
+
printf("%#{spacing}d %s\n".colorize(:red), line_number, line_content)
|
39
|
+
else
|
40
|
+
printf("%#{spacing}d %s\n", line_number, line_content)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
exit 1 unless ENV['TEST']
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def set_template_variables(variables)
|
48
|
+
variables.each do |key, value|
|
49
|
+
instance_variable_set(:"@#{key}", value)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,291 @@
|
|
1
|
+
require "active_support/core_ext/hash"
|
2
|
+
require "aws-sdk-dynamodb"
|
3
|
+
require "digest"
|
4
|
+
require "yaml"
|
5
|
+
|
6
|
+
# The modeling is ActiveRecord-ish but not exactly because DynamoDB is a
|
7
|
+
# different type of database.
|
8
|
+
#
|
9
|
+
# Examples:
|
10
|
+
#
|
11
|
+
# post = MyModel.new
|
12
|
+
# post = post.replace(title: "test title")
|
13
|
+
#
|
14
|
+
# post.attrs[:id] now contain a generaetd unique partition_key id.
|
15
|
+
# Usually the partition_key is 'id'. You can set your own unique id also:
|
16
|
+
#
|
17
|
+
# post = MyModel.new(id: "myid", title: "my title")
|
18
|
+
# post.replace
|
19
|
+
#
|
20
|
+
# Note that the replace method replaces the entire item, so you
|
21
|
+
# need to merge the attributes if you want to keep the other attributes.
|
22
|
+
#
|
23
|
+
# post = MyModel.find("myid")
|
24
|
+
# post.attrs = post.attrs.deep_merge("desc": "my desc") # keeps title field
|
25
|
+
# post.replace
|
26
|
+
#
|
27
|
+
# The convenience `attrs` method performs a deep_merge:
|
28
|
+
#
|
29
|
+
# post = MyModel.find("myid")
|
30
|
+
# post.attrs("desc": "my desc") # <= does a deep_merge
|
31
|
+
# post.replace
|
32
|
+
#
|
33
|
+
# Note, a race condition edge case can exist when several concurrent replace
|
34
|
+
# calls are happening. This is why the interface is called replace to
|
35
|
+
# emphasis that possibility.
|
36
|
+
# TODO: implement post.update with db.update_item in a Ruby-ish way.
|
37
|
+
#
|
38
|
+
module Dynomite
|
39
|
+
class Item
|
40
|
+
include Log
|
41
|
+
include DbConfig
|
42
|
+
|
43
|
+
def initialize(attrs={})
|
44
|
+
@attrs = attrs
|
45
|
+
end
|
46
|
+
|
47
|
+
# Defining our own reader so we can do a deep merge if user passes in attrs
|
48
|
+
def attrs(*args)
|
49
|
+
case args.size
|
50
|
+
when 0
|
51
|
+
ActiveSupport::HashWithIndifferentAccess.new(@attrs)
|
52
|
+
when 1
|
53
|
+
attributes = args[0] # Hash
|
54
|
+
if attributes.empty?
|
55
|
+
ActiveSupport::HashWithIndifferentAccess.new
|
56
|
+
else
|
57
|
+
@attrs = attrs.deep_merge!(attributes)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# Not using method_missing to allow usage of dot notation and assign
|
63
|
+
# @attrs because it might hide actual missing methods errors.
|
64
|
+
# DynamoDB attrs can go many levels deep so it makes less make sense to
|
65
|
+
# use to dot notation.
|
66
|
+
|
67
|
+
# The method is named replace to clearly indicate that the item is
|
68
|
+
# fully replaced.
|
69
|
+
def replace
|
70
|
+
attrs = self.class.replace(@attrs)
|
71
|
+
@attrs = attrs # refresh attrs because it now has the id
|
72
|
+
end
|
73
|
+
|
74
|
+
def find(id)
|
75
|
+
self.class.find(id)
|
76
|
+
end
|
77
|
+
|
78
|
+
def delete
|
79
|
+
self.class.delete(@attrs[:id]) if @attrs[:id]
|
80
|
+
end
|
81
|
+
|
82
|
+
def table_name
|
83
|
+
self.class.table_name
|
84
|
+
end
|
85
|
+
|
86
|
+
def partition_key
|
87
|
+
self.class.partition_key
|
88
|
+
end
|
89
|
+
|
90
|
+
# For render json: item
|
91
|
+
def as_json(options={})
|
92
|
+
@attrs
|
93
|
+
end
|
94
|
+
|
95
|
+
# Longer hand methods for completeness.
|
96
|
+
# Internallly encourage the shorter attrs method.
|
97
|
+
def attributes=(attributes)
|
98
|
+
@attributes = attributes
|
99
|
+
end
|
100
|
+
|
101
|
+
def attributes
|
102
|
+
@attributes
|
103
|
+
end
|
104
|
+
|
105
|
+
# Adds very little wrapper logic to scan.
|
106
|
+
#
|
107
|
+
# * Automatically add table_name to options for convenience.
|
108
|
+
# * Decorates return value. Returns Array of [MyModel.new] instead of the
|
109
|
+
# dynamodb client response.
|
110
|
+
#
|
111
|
+
# Other than that, usage is same was using the dynamodb client scan method
|
112
|
+
# directly. Example:
|
113
|
+
#
|
114
|
+
# MyModel.scan(
|
115
|
+
# expression_attribute_names: {"#updated_at"=>"updated_at"},
|
116
|
+
# filter_expression: "#updated_at between :start_time and :end_time",
|
117
|
+
# expression_attribute_values: {
|
118
|
+
# ":start_time" => "2010-01-01T00:00:00",
|
119
|
+
# ":end_time" => "2020-01-01T00:00:00"
|
120
|
+
# }
|
121
|
+
# )
|
122
|
+
#
|
123
|
+
# AWS Docs examples: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.Ruby.04.html
|
124
|
+
def self.scan(params={})
|
125
|
+
log("It's recommended to not use scan for production. It can be slow and expensive. You can a LSI or GSI and query the index instead.")
|
126
|
+
params = { table_name: table_name }.merge(params)
|
127
|
+
resp = db.scan(params)
|
128
|
+
resp.items.map {|i| self.new(i) }
|
129
|
+
end
|
130
|
+
|
131
|
+
# Adds very little wrapper logic to query.
|
132
|
+
#
|
133
|
+
# * Automatically add table_name to options for convenience.
|
134
|
+
# * Decorates return value. Returns Array of [MyModel.new] instead of the
|
135
|
+
# dynamodb client response.
|
136
|
+
#
|
137
|
+
# Other than that, usage is same was using the dynamodb client query method
|
138
|
+
# directly. Example:
|
139
|
+
#
|
140
|
+
# MyModel.query(
|
141
|
+
# index_name: 'category-index',
|
142
|
+
# expression_attribute_names: { "#category_name" => "category" },
|
143
|
+
# expression_attribute_values: { ":category_value" => "Entertainment" },
|
144
|
+
# key_condition_expression: "#category_name = :category_value",
|
145
|
+
# )
|
146
|
+
#
|
147
|
+
# AWS Docs examples: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.Ruby.04.html
|
148
|
+
def self.query(params={})
|
149
|
+
params = { table_name: table_name }.merge(params)
|
150
|
+
resp = db.query(params)
|
151
|
+
resp.items.map {|i| self.new(i) }
|
152
|
+
end
|
153
|
+
|
154
|
+
# Translates simple query searches:
|
155
|
+
#
|
156
|
+
# Post.where({category: "Drama"}, index_name: "category-index")
|
157
|
+
#
|
158
|
+
# translates to
|
159
|
+
#
|
160
|
+
# resp = db.query(
|
161
|
+
# table_name: "demo-dev-post",
|
162
|
+
# index_name: 'category-index',
|
163
|
+
# expression_attribute_names: { "#category_name" => "category" },
|
164
|
+
# expression_attribute_values: { ":category_value" => category },
|
165
|
+
# key_condition_expression: "#category_name = :category_value",
|
166
|
+
# )
|
167
|
+
#
|
168
|
+
# TODO: Implement nicer where syntax with index_name as a chained method.
|
169
|
+
#
|
170
|
+
# Post.where({category: "Drama"}, {index_name: "category-index"})
|
171
|
+
# VS
|
172
|
+
# Post.where(category: "Drama").index_name("category-index")
|
173
|
+
def self.where(attributes, options={})
|
174
|
+
raise "attributes.size == 1 only supported for now" if attributes.size != 1
|
175
|
+
|
176
|
+
attr_name = attributes.keys.first
|
177
|
+
attr_value = attributes[attr_name]
|
178
|
+
|
179
|
+
# params = {
|
180
|
+
# expression_attribute_names: { "#category_name" => "category" },
|
181
|
+
# expression_attribute_values: { ":category_value" => "Entertainment" },
|
182
|
+
# key_condition_expression: "#category_name = :category_value",
|
183
|
+
# }
|
184
|
+
name_key, value_key = "##{attr_name}_name", ":#{attr_name}_value"
|
185
|
+
params = {
|
186
|
+
expression_attribute_names: { name_key => attr_name },
|
187
|
+
expression_attribute_values: { value_key => attr_value },
|
188
|
+
key_condition_expression: "#{name_key} = #{value_key}",
|
189
|
+
}
|
190
|
+
# Allow direct access to override params passed to dynamodb query options.
|
191
|
+
# This is is how index_name is passed:
|
192
|
+
params = params.merge(options)
|
193
|
+
|
194
|
+
query(params)
|
195
|
+
end
|
196
|
+
|
197
|
+
def self.replace(attrs)
|
198
|
+
# Automatically adds some attributes:
|
199
|
+
# partition key unique id
|
200
|
+
# created_at and updated_at timestamps. Timestamp format from AWS docs: http://amzn.to/2z98Bdc
|
201
|
+
defaults = {
|
202
|
+
partition_key => Digest::SHA1.hexdigest([Time.now, rand].join)
|
203
|
+
}
|
204
|
+
item = defaults.merge(attrs)
|
205
|
+
item["created_at"] ||= Time.now.utc.strftime('%Y-%m-%dT%TZ')
|
206
|
+
item["updated_at"] = Time.now.utc.strftime('%Y-%m-%dT%TZ')
|
207
|
+
|
208
|
+
# put_item full replaces the item
|
209
|
+
resp = db.put_item(
|
210
|
+
table_name: table_name,
|
211
|
+
item: item
|
212
|
+
)
|
213
|
+
|
214
|
+
# The resp does not contain the attrs. So might as well return
|
215
|
+
# the original item with the generated partition_key value
|
216
|
+
item
|
217
|
+
end
|
218
|
+
|
219
|
+
def self.find(id)
|
220
|
+
resp = db.get_item(
|
221
|
+
table_name: table_name,
|
222
|
+
key: {partition_key => id}
|
223
|
+
)
|
224
|
+
attributes = resp.item # unwraps the item's attributes
|
225
|
+
self.new(attributes) if attributes
|
226
|
+
end
|
227
|
+
|
228
|
+
# Two ways to use the delete method:
|
229
|
+
#
|
230
|
+
# 1. Specify the key as a String. In this case the key will is the partition_key
|
231
|
+
# set on the model.
|
232
|
+
# MyModel.delete("728e7b5df40b93c3ea6407da8ac3e520e00d7351")
|
233
|
+
#
|
234
|
+
# 2. Specify the key as a Hash, you can arbitrarily specific the key structure this way
|
235
|
+
# MyModel.delete("728e7b5df40b93c3ea6407da8ac3e520e00d7351")
|
236
|
+
#
|
237
|
+
# options is provided in case you want to specific condition_expression or
|
238
|
+
# expression_attribute_values.
|
239
|
+
def self.delete(key_object, options={})
|
240
|
+
if key_object.is_a?(String)
|
241
|
+
key = {
|
242
|
+
partition_key => key_object
|
243
|
+
}
|
244
|
+
else # it should be a Hash
|
245
|
+
key = key_object
|
246
|
+
end
|
247
|
+
|
248
|
+
params = {
|
249
|
+
table_name: table_name,
|
250
|
+
key: key
|
251
|
+
}
|
252
|
+
# In case you want to specify condition_expression or expression_attribute_values
|
253
|
+
params = params.merge(options)
|
254
|
+
|
255
|
+
resp = db.delete_item(params)
|
256
|
+
end
|
257
|
+
|
258
|
+
# When called with an argument we'll set the internal @partition_key value
|
259
|
+
# When called without an argument just retun it.
|
260
|
+
# class Comment < Dynomite::Item
|
261
|
+
# partition_key "post_id"
|
262
|
+
# end
|
263
|
+
def self.partition_key(*args)
|
264
|
+
case args.size
|
265
|
+
when 0
|
266
|
+
@partition_key || "id" # defaults to id
|
267
|
+
when 1
|
268
|
+
@partition_key = args[0].to_s
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
def self.table_name(*args)
|
273
|
+
case args.size
|
274
|
+
when 0
|
275
|
+
get_table_name
|
276
|
+
when 1
|
277
|
+
set_table_name(args[0])
|
278
|
+
end
|
279
|
+
end
|
280
|
+
|
281
|
+
def self.get_table_name
|
282
|
+
@table_name ||= self.name.pluralize.underscore
|
283
|
+
[table_namespace, @table_name].reject {|s| s.nil? || s.empty?}.join('-')
|
284
|
+
end
|
285
|
+
|
286
|
+
def self.set_table_name(value)
|
287
|
+
@table_name = value
|
288
|
+
end
|
289
|
+
|
290
|
+
end
|
291
|
+
end
|